Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-1j5f0 Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-1j5f0 is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-n0bd1-tw1dh --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8b9ec29238826bd90be327024c32d72c47721282" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-n0bd1" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-1j5f0" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-1j5f0" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-1j5f0" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-1j5f0 in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" First time build. Skipping changelog. [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-list --no-walk a0a6bb5614d55afbab88f0d1ae459423175a5407 # timeout=10 + printenv PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=b309b4a0-d4d0-473d-9ce8-7ca4d27cfae4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-1j5f0 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-n0bd1 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-6317e49050c5338128550ce78b4e3dd1c31eef0baa721258868438c2a09fc672 NODE_LABELS=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-n0bd1 cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-1j5f0 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-1j5f0 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.6' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build2196450379=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-1j5f0 bash' debug command: kubectl -n jenkins-tiflow exec -ti cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-1j5f0 bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 34799 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: 1962ebffeb4d86f0 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 25 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Mon May 06 11:32:53 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11026/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11026/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11026/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11026/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [examples/java/pom.xml] [Pipeline] echo diff file not matched: examples/java/pom.xml [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-936b4d4) 203908608 bytes in 2.61 secs (78203690 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/.git/ .git HEAD is now at 936b4d4d2 kafka(ticdc): Fix java example license issue (#11024) POST git-upload-pack (686 bytes) POST git-upload-pack (1023 bytes) POST git-upload-pack (gzip 2173 to 1144 bytes) From https://github.com/pingcap/tiflow 3582359ab..38eb04081 release-8.1 -> origin/release-8.1 * [new ref] refs/pull/11026/head -> origin/pr/11026/head Previous HEAD position was 936b4d4d2 kafka(ticdc): Fix java example license issue (#11024) HEAD is now at 38eb04081 *(ticdc): Initialize changefeed/processor asynchronously (#10832) (#10985) ๐Ÿšง Checkouting to base SHA:38eb040812f817df410abe870952629df60fd01f... HEAD is now at 38eb04081 *(ticdc): Initialize changefeed/processor asynchronously (#10832) (#10985) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: 38eb040812f817df410abe870952629df60fd01f 38eb04081 *(ticdc): Initialize changefeed/processor asynchronously (#10832) (#10985) c78adb6f1 codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901) (#11002) 76cd91300 cli(ticdc): allow client authentication to be enabled without tls (#11005) (#11016) ๐Ÿšง Pre-merge heads of pull requests to base SHA: 38eb040812f817df410abe870952629df60fd01f ... Updating 38eb04081..e1a27adc6 Fast-forward examples/java/pom.xml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) ๐Ÿงพ Pre-merged result: e1a27adc69acc9d50db0c3fea2945bbd6d983e30 e1a27adc6 fix java example license issue 38eb04081 *(ticdc): Initialize changefeed/processor asynchronously (#10832) (#10985) c78adb6f1 codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901) (#11002) โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache saved successfully (git/pingcap/tiflow/rev-38eb040-e1a27ad) 204049920 bytes in 3.38 secs (60454020 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./scripts/download-integration-test-binaries.sh release-8.1 Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 679 0 --:--:-- --:--:-- --:--:-- 683 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1227 0 --:--:-- --:--:-- --:--:-- 1242 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 624 0 --:--:-- --:--:-- --:--:-- 621 100 41 100 41 0 0 624 0 --:--:-- --:--:-- --:--:-- 621 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 342 0 --:--:-- --:--:-- --:--:-- 344 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/c8e6662d10deed030ef864169ae27718f5a61991/centos7/tidb-server.tar.gz 2024-05-06 19:33:27 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/c8e6662d10deed030ef864169ae27718f5a61991/centos7/tidb-server.tar.gz [526223028/526223028] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/13eaf8aaa661dc0d3885ac9e59145f69752cee2d/centos7/pd-server.tar.gz 2024-05-06 19:33:44 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/13eaf8aaa661dc0d3885ac9e59145f69752cee2d/centos7/pd-server.tar.gz [184187328/184187328] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/56613f7c3e28c02853cc51d15bc1b77f68b58be8/centos7/tikv-server.tar.gz 2024-05-06 19:34:24 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/56613f7c3e28c02853cc51d15bc1b77f68b58be8/centos7/tikv-server.tar.gz [834867730/834867730] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/release-8.1/eb96e720183ea798458afc99cfb034864ed75c4c/centos7/tiflash.tar.gz 2024-05-06 19:34:55 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/release-8.1/eb96e720183ea798458afc99cfb034864ed75c4c/centos7/tiflash.tar.gz [455317481/455317481] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-06 19:35:01 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-05-06 19:35:04 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-05-06 19:35:05 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-05-06 19:35:06 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz 2024-05-06 19:35:10 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz [79877126/79877126] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> download schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2024-05-06 19:35:23 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 6 19:35 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 19:35 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 6 19:35 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 6 19:35 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 5 14:57 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 5 14:57 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 5 14:57 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 5 14:57 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 5 14:18 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M May 5 14:58 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 15:24 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 15:24 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 15:24 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 15:24 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 15:23 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 15:24 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 15:24 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 15:24 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 205M May 6 18:40 tidb-server -rwxr-xr-x. 1 jenkins jenkins 379M May 5 14:58 tiflash -rwxr-xr-x. 1 jenkins jenkins 367M Apr 30 15:02 tikv-server + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ./bin/ + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 6 19:35 . drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 19:35 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 6 19:35 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 6 19:35 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 5 14:57 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 5 14:57 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 5 14:57 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 5 14:57 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 5 14:18 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M May 5 14:58 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 15:24 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 15:24 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 15:24 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 15:24 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 15:23 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 15:24 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 15:24 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 15:24 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 205M May 6 18:40 tidb-server -rwxr-xr-x. 1 jenkins jenkins 379M May 5 14:58 tiflash -rwxr-xr-x. 1 jenkins jenkins 367M Apr 30 15:02 tikv-server + ./bin/tidb-server -V Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 + ./bin/tikv-server -V TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.4.0 Git Commit Hash: d671b0840063bc2532941f02e02e12627402844c Git Branch: heads/refs/tags/v7.4.0 UTC Build Time: 2023-09-22 03:51:56 Go Version: go1.21.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x. 2 jenkins jenkins 4.0K May 6 19:35 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 19:35 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.1.0-alpha-23-ge1a27adc6" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 11:35:26" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e1a27adc69acc9d50db0c3fea2945bbd6d983e30" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.1.0-alpha-23-ge1a27adc6"' -o bin/cdc ./cmd/cdc go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/google/uuid v1.6.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/stretchr/testify v1.9.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/YangKeao/seahash v0.0.0-20240229041150-e7bf269c3140 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/goccy/go-json v0.10.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading github.com/swaggo/swag v1.16.3 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/glebarez/sqlite v1.7.0 go: downloading gorm.io/driver/mysql v1.3.3 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/go-openapi/spec v0.21.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/glebarez/go-sqlite v1.21.2 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading modernc.org/libc v1.37.1 go: downloading modernc.org/sqlite v1.27.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/go-openapi/jsonpointer v0.21.0 go: downloading github.com/go-openapi/jsonreference v0.21.0 go: downloading github.com/go-openapi/swag v0.23.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading k8s.io/apimachinery v0.28.6 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading modernc.org/memory v1.7.2 go: downloading modernc.org/mathutil v1.6.0 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + '[' -f ./bin/cdc_kafka_consumer ']' + make kafka_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.1.0-alpha-23-ge1a27adc6" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 11:38:23" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e1a27adc69acc9d50db0c3fea2945bbd6d983e30" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.1.0-alpha-23-ge1a27adc6"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go + '[' -f ./bin/cdc_storage_consumer ']' + make storage_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.1.0-alpha-23-ge1a27adc6" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 11:38:30" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e1a27adc69acc9d50db0c3fea2945bbd6d983e30" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.1.0-alpha-23-ge1a27adc6"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.1.0-alpha-23-ge1a27adc6" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 11:38:38" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e1a27adc69acc9d50db0c3fea2945bbd6d983e30" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.1.0-alpha-23-ge1a27adc6"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.1.0-alpha-23-ge1a27adc6" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 11:38:38" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e1a27adc69acc9d50db0c3fea2945bbd6d983e30" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.1.0-alpha-23-ge1a27adc6"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.1.0-alpha-23-ge1a27adc6" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 11:38:38" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e1a27adc69acc9d50db0c3fea2945bbd6d983e30" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.1.0-alpha-23-ge1a27adc6"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.1.0-alpha-23-ge1a27adc6" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 11:38:38" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e1a27adc69acc9d50db0c3fea2945bbd6d983e30" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.1.0-alpha-23-ge1a27adc6"' -o bin/oauth2-server ./cmd/oauth2-server/main.go go: downloading github.com/go-oauth2/oauth2/v4 v4.5.2 go: downloading github.com/tidwall/buntdb v1.3.0 go: downloading github.com/tidwall/rtred v0.1.2 go: downloading github.com/tidwall/match v1.1.1 go: downloading github.com/tidwall/grect v0.1.4 go: downloading github.com/tidwall/gjson v1.14.3 go: downloading github.com/tidwall/tinyqueue v0.1.1 go: downloading github.com/tidwall/pretty v1.2.0 $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20231213065948-336e064b488d go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading go.uber.org/dig v1.13.0 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading go.uber.org/goleak v1.3.0 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.1.0-alpha-23-ge1a27adc6" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 11:38:38" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e1a27adc69acc9d50db0c3fea2945bbd6d983e30" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.1.0-alpha-23-ge1a27adc6"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.1.0-alpha-23-ge1a27adc6" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 11:38:38" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e1a27adc69acc9d50db0c3fea2945bbd6d983e30" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.1.0-alpha-23-ge1a27adc6"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.2G drwxr-sr-x. 2 jenkins jenkins 4.0K May 6 19:43 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 19:35 .. -rwxr-xr-x. 1 jenkins jenkins 220M May 6 19:43 cdc -rwxr-xr-x. 1 jenkins jenkins 363M May 6 19:43 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 6 19:38 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 6 19:39 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 6 19:38 cdc_storage_consumer -rwxr-xr-x. 1 jenkins jenkins 12M May 6 19:39 oauth2-server + ./bin/cdc version Release Version: v8.1.0-alpha-23-ge1a27adc6 Git Commit Hash: e1a27adc69acc9d50db0c3fea2945bbd6d983e30 Git Branch: HEAD UTC Build Time: 2024-05-06 11:38:38 Go Version: go version go1.21.6 linux/amd64 Failpoint Build: true [Pipeline] } Cache saved successfully (binary/pingcap/tiflow/cdc-integration-test/rev-38eb040-e1a27ad) 1194936832 bytes in 31.67 secs (37727116 bytes/sec) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ./bin/ + ls -alh ./bin total 3.0G drwxr-sr-x. 6 jenkins jenkins 4.0K May 6 19:44 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 19:35 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 6 19:44 bin -rwxr-xr-x. 1 jenkins jenkins 220M May 6 19:43 cdc -rwxr-xr-x. 1 jenkins jenkins 363M May 6 19:43 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 6 19:38 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 6 19:39 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 6 19:38 cdc_storage_consumer drwxr-sr-x. 4 jenkins jenkins 4.0K May 6 19:44 etc -rwxr-xr-x. 1 jenkins jenkins 17M May 6 19:44 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 6 19:44 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 6 19:44 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 19:44 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 6 19:44 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K May 6 19:44 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 6 19:44 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K May 6 19:44 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 6 19:44 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 6 19:44 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 6 19:44 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M May 6 19:44 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M May 6 19:44 minio -rwxr-xr-x. 1 jenkins jenkins 12M May 6 19:39 oauth2-server -rwxr-xr-x. 1 jenkins jenkins 37M May 6 19:44 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M May 6 19:44 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M May 6 19:44 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M May 6 19:44 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M May 6 19:44 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M May 6 19:44 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 32M May 6 19:44 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 6 19:44 share -rwxr-xr-x. 1 jenkins jenkins 32M May 6 19:44 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M May 6 19:44 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 205M May 6 19:44 tidb-server -rwxr-xr-x. 1 jenkins jenkins 379M May 6 19:44 tiflash -rwxr-xr-x. 1 jenkins jenkins 367M May 6 19:44 tikv-server [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 103.21 secs (35539266 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G18') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G19') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G20') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G21') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G18') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G19') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G20') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G21') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9 [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-mzzb9 Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl [Pipeline] node Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-lmt9j-79hhf --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "354bd387bfa78e85b9da5cc90053a40230a45dad" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-lmt9j" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3n4n5 Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9 is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-c92td-8qtzq --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8902404556618cb9346cafaa2327c464cbecb39b" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-c92td" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9 in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7bcc65c9; decorates RemoteLauncher[hudson.remoting.Channel@49877e57:JNLP4-connect connection from 10.233.70.66/10.233.70.66:38790] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4 Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@571ac999; decorates RemoteLauncher[hudson.remoting.Channel@6b68a5ff:JNLP4-connect connection from 10.233.107.187/10.233.107.187:43836] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84 Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-1stjh-d40c1 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "df9d51e6351469bc0ca48727bf3f6e1385444435" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-1stjh" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test [Pipeline] { [Pipeline] checkout Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-gjc6p-895tk --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2057d7ac466c04a1d407f1492576069a488de4b5" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-gjc6p" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Avoid second fetch The recommended git tool is: git Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test [Pipeline] { Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72 [Pipeline] node [Pipeline] checkout The recommended git tool is: git Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-zf4c9-78xx3 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a8815e91528371572f765ddafafc753bec10f567" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-zf4c9" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test [Pipeline] { [Pipeline] checkout [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { The recommended git tool is: git [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] cache Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@30bbd471; decorates RemoteLauncher[hudson.remoting.Channel@22c8be57:JNLP4-connect connection from 10.233.90.84/10.233.90.84:54690] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3749ea75; decorates RemoteLauncher[hudson.remoting.Channel@41892fda:JNLP4-connect connection from 10.233.67.98/10.233.67.98:39192] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3de85a7b; decorates RemoteLauncher[hudson.remoting.Channel@52ff59e8:JNLP4-connect connection from 10.233.70.62/10.233.70.62:55142] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4 is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-3z0ms-8pqgf Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3n4n5 is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-bltlv-nw5r4 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "783e10353526cf71227ce8da265d2c1d6b6ebae1" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-3z0ms" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false --- apiVersion: "v1" kind: "Pod" metadata: annotations: - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8e05e76bc6a230de8d3b4b2725c207e8c4fbaa38" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-bltlv" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3n4n5" namespace: "jenkins-tiflow" spec: affinity: - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4" nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3n4n5" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3n4n5" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4 in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3n4n5 in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-mzzb9 is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-gx9jn-2ht3p --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "750f1fa003cdbae92afdd90814673b8f4ce4348b" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-gx9jn" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-mzzb9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-mzzb9" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-mzzb9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-mzzb9 in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72 is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-dk00k-s0874 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "cf04d717ac013644a7eac102966211ac347d7806" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-dk00k" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72 in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84 is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-7312q-mtgv4 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9769a4563cca45366f609ff0bfb26251f3fb003c" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-7312q" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84 in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-qsh8g-5h1zv --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "170cdda59706949b40b057b25e6cab94d280b8be" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-qsh8g" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Obtained pipelines/pingcap/tiflow/release-8.1/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-l18q5-mzq5x --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "078ec6d89f9dd3770b97ddabed9179740fd7312d" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-l18q5" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 14.55 secs (252128436 bytes/sec) [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] { [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G00 Run cases: bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_suicide server_config_compatibility kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro debezium lossy_ddl storage_csv_update PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=2db3df1f-56cb-4b5e-82f3-b53ac4f14f07 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-lmt9j GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-lmt9j cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] { The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] { The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-54cks [Pipeline] timeout Timeout set to expire in 40 min Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-fd5nv [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-98cvh [Pipeline] timeout Timeout set to expire in 40 min Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-rqt30 [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-dwwvr [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-b2lwz Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-c3sn1 Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-5k7gg Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-89kh8 Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN Created Pod: kubernetes jenkins-tiflow/cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-59v7t [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1ae5e49b; decorates RemoteLauncher[hudson.remoting.Channel@5b62cf94:JNLP4-connect connection from 10.233.105.155/10.233.105.155:55596] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2ed7862c; decorates RemoteLauncher[hudson.remoting.Channel@5991e181:JNLP4-connect connection from 10.233.72.95/10.233.72.95:58672] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4a826cfc; decorates RemoteLauncher[hudson.remoting.Channel@7c28bd43:JNLP4-connect connection from 10.233.70.206/10.233.70.206:41642] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5ea24761; decorates RemoteLauncher[hudson.remoting.Channel@e89b390:JNLP4-connect connection from 10.233.100.116/10.233.100.116:42298] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5f373100; decorates RemoteLauncher[hudson.remoting.Channel@4f040ab:JNLP4-connect connection from 10.233.66.4/10.233.66.4:52028] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7f025965; decorates RemoteLauncher[hudson.remoting.Channel@307d053:JNLP4-connect connection from 10.233.67.54/10.233.67.54:36334] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@18429de1; decorates RemoteLauncher[hudson.remoting.Channel@490de5ef:JNLP4-connect connection from 10.233.71.90/10.233.71.90:32890] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 start tidb cluster in /tmp/tidb_cdc_test/bdr_mode Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-54cks is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-gl74w-tk8hj --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "851281e27faaed35c60589772289f044927ed9d4" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-gl74w" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-54cks" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-54cks" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-54cks" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-54cks in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-rqt30 is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-m70kp-cwnv7 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "037097bee5d10f182704238ef8f4163ce3ca3e81" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-m70kp" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-rqt30" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-rqt30" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-rqt30" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-rqt30 in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-89kh8 is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-w3fs6-nh25f --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "abcc54107a72d0be15ed062c755e920d0aa627dd" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-w3fs6" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-89kh8" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-89kh8" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-89kh8" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-89kh8 in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-5k7gg is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-l3m7x-3b363 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "de19855656cb57e427a802dde9524508327bd5f2" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-l3m7x" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-5k7gg" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-5k7gg" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-5k7gg" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-59v7t is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-5t6wm-mb5kf --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "5d86133b6f91050a44d80b4a95221b788e1a02fd" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-5t6wm" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-59v7t" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-59v7t" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-59v7t" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-5k7gg in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-59v7t in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-dwwvr is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-j5rm5-k67t7 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "4d4c6b3aa5ad4236b3f6199e1a4ac2f9791417dd" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-j5rm5" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-dwwvr" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-dwwvr" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-dwwvr" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-dwwvr in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-b2lwz is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-p11tp-v5k2k --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7a6daf454d69a9dc9c40fea4ebff0b47b1aec7e9" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-p11tp" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-b2lwz" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-b2lwz" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-b2lwz" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-b2lwz in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-c3sn1 is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-cdqw4-d7v84 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "74433216df4795bc5d4494470ded134389f8effd" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-cdqw4" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-c3sn1" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-c3sn1" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-c3sn1" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-c3sn1 in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-fd5nv is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-821f0-gbnn7 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "46b7d81793ae4f879faabeced1ff159f38bdfda8" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-821f0" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-fd5nv" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-fd5nv" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-fd5nv" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-fd5nv in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 5.76 secs (636885722 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36e9c6ac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl, pid:1165, start at 2024-05-06 19:46:55.576206648 +0800 CST m=+5.400297109 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:48:55.583 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:46:55.581 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:36:55.581 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Agent cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-98cvh is provisioned from template cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-57vfd-vtjc6 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" runUrl: "job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "adc8a5a454d1dbd600cd517d301b9b0a71790473" jenkins/label: "cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-57vfd" name: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-98cvh" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-98cvh" - name: "JENKINS_NAME" value: "cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-98cvh" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-98cvh in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36e9c6ac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl, pid:1165, start at 2024-05-06 19:46:55.576206648 +0800 CST m=+5.400297109 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:48:55.583 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:46:55.581 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:36:55.581 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36e9c75c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl, pid:1252, start at 2024-05-06 19:46:55.591137839 +0800 CST m=+5.309361094 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:48:55.599 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:46:55.575 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:36:55.575 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 19:47:00 CST 2024] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.26062608.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:47:03 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dcb051b9-3f7a-440f-854a-68afecb901c4 {"id":"dcb051b9-3f7a-440f-854a-68afecb901c4","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996020} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dba4cbc31 dcb051b9-3f7a-440f-854a-68afecb901c4 /tidb/cdc/default/default/upstream/7365851764145360313 {"id":7365851764145360313,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dcb051b9-3f7a-440f-854a-68afecb901c4 {"id":"dcb051b9-3f7a-440f-854a-68afecb901c4","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996020} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dba4cbc31 dcb051b9-3f7a-440f-854a-68afecb901c4 /tidb/cdc/default/default/upstream/7365851764145360313 {"id":7365851764145360313,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dcb051b9-3f7a-440f-854a-68afecb901c4 {"id":"dcb051b9-3f7a-440f-854a-68afecb901c4","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996020} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dba4cbc31 dcb051b9-3f7a-440f-854a-68afecb901c4 /tidb/cdc/default/default/upstream/7365851764145360313 {"id":7365851764145360313,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:47:03 CST 2024] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.26642666.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdcdown.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_datadown --cluster-id default --addr 127.0.0.1:8400 --pd http://127.0.0.1:2479 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8400; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8400 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8400 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:47:06 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7009c23-1a50-4a47-af84-f2a3e10ab40d {"id":"c7009c23-1a50-4a47-af84-f2a3e10ab40d","address":"127.0.0.1:8400","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996023} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118f4dba4ce655 c7009c23-1a50-4a47-af84-f2a3e10ab40d /tidb/cdc/default/default/upstream/7365851761564073086 {"id":7365851761564073086,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7009c23-1a50-4a47-af84-f2a3e10ab40d {"id":"c7009c23-1a50-4a47-af84-f2a3e10ab40d","address":"127.0.0.1:8400","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996023} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118f4dba4ce655 c7009c23-1a50-4a47-af84-f2a3e10ab40d /tidb/cdc/default/default/upstream/7365851761564073086 {"id":7365851761564073086,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7009c23-1a50-4a47-af84-f2a3e10ab40d {"id":"c7009c23-1a50-4a47-af84-f2a3e10ab40d","address":"127.0.0.1:8400","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996023} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118f4dba4ce655 c7009c23-1a50-4a47-af84-f2a3e10ab40d /tidb/cdc/default/default/upstream/7365851761564073086 {"id":7365851761564073086,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.2716.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306 -c test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/bdr_mode/conf/up.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365851764145360313,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306","create_time":"2024-05-06T19:47:07.17314289+08:00","start_ts":449575918510014469,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"enable_table_monitor":false,"bdr_mode":true,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575918510014469,"checkpoint_ts":449575918510014469,"checkpoint_time":"2024-05-06 19:47:07.031"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.2747.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:4000 -c test-2 --server http://127.0.0.1:8400 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/bdr_mode/conf/down.toml Create changefeed successfully! ID: test-2 Info: {"upstream_id":7365851761564073086,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:4000","create_time":"2024-05-06T19:47:09.113997779+08:00","start_ts":449575919019884549,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":true,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575919019884549,"checkpoint_ts":449575919019884549,"checkpoint_time":"2024-05-06 19:47:08.976"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + set +x Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 17.13 secs (214144387 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 13.96 secs (262782912 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 15.53 secs (236119870 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G01 Run cases: http_api http_api_tls api_v2 http_api_tls_with_user_auth cli_tls_with_auth kafka_simple_basic kafka_simple_basic_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_simple_claim_check kafka_simple_claim_check_avro canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible multi_topics avro_basic canal_json_handle_key_only open_protocol_handle_key_only canal_json_claim_check open_protocol_claim_check canal_json_storage_basic canal_json_storage_partition_table multi_tables_ddl PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=a7327d1f-6ea2-4abf-97e1-a79e35aeff3e BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-c92td GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-c92td cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Collecting requests==2.26.0 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=baee2558-a975-4ab7-8754-6b0406b55365 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-1stjh GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-1stjh cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G04 Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=92bcba85-5f2d-42c2-a1b7-85195d333710 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-gjc6p GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-gjc6p cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G03 Run cases: row_format drop_many_tables processor_stop_delay partition_table PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=19ae3e2c-ae3e-4034-92b7-9337e49e929e BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-zf4c9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-zf4c9 cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { Downloading https://files.pythonhosted.org/packages/92/96/144f70b972a9c0eabbd4391ef93ccd49d0f2747f4f6a2a2738e99e5adc65/requests-2.26.0-py2.py3-none-any.whl (62kB) [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { table tidb_cdc.syncpoint_v1 exists table tidb_cdc.syncpoint_v1 does not exists table bdr_mode.finish_mark exists check diff failed 1-th time, retry later [Pipeline] { Collecting urllib3<1.27,>=1.21.1 (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/b0/53/aa91e163dcfd1e5b82d8a890ecf13314e3e149c05270cc644581f77f17fd/urllib3-1.26.18-py2.py3-none-any.whl (143kB) [Pipeline] checkout [Pipeline] { The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) check diff failed 2-th time, retry later No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7acbf560; decorates RemoteLauncher[hudson.remoting.Channel@1ebe1171:JNLP4-connect connection from 10.233.86.119/10.233.86.119:48300] will be ignored (a typical symptom is the Git executable not being run inside a designated container) [Pipeline] timeout Cloning the remote Git repository Using shallow clone with depth 1 Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Collecting certifi>=2017.4.17 (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/ba/06/a07f096c664aeb9f01624f858c3add0a4e913d6c96257acb4fce61e7de14/certifi-2024.2.2-py3-none-any.whl (163kB) Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@504269a5; decorates RemoteLauncher[hudson.remoting.Channel@1a2b69bd:JNLP4-connect connection from 10.233.126.14/10.233.126.14:36436] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4a2f038f; decorates RemoteLauncher[hudson.remoting.Channel@53b5feb7:JNLP4-connect connection from 10.233.68.218/10.233.68.218:39692] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@517b4ac7; decorates RemoteLauncher[hudson.remoting.Channel@4ff05270:JNLP4-connect connection from 10.233.71.41/10.233.71.41:36972] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Collecting idna<4,>=2.5; python_version >= "3" (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/e5/3e/741d8c82801c347547f8a2a06aa57dbb1992be9e948df2ea0eda2c8b79e8/idna-3.7-py3-none-any.whl (66kB) Installing collected packages: urllib3, certifi, idna, requests Found existing installation: urllib3 1.26.16 [Pipeline] { [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4dc079fa; decorates RemoteLauncher[hudson.remoting.Channel@612ea96e:JNLP4-connect connection from 10.233.67.243/10.233.67.243:60048] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@20b707c2; decorates RemoteLauncher[hudson.remoting.Channel@69164aa6:JNLP4-connect connection from 10.233.72.65/10.233.72.65:57158] will be ignored (a typical symptom is the Git executable not being run inside a designated container) [Pipeline] { Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@637767ed; decorates RemoteLauncher[hudson.remoting.Channel@7f0132b8:JNLP4-connect connection from 10.233.105.64/10.233.105.64:44976] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6f060e79; decorates RemoteLauncher[hudson.remoting.Channel@53c9dc17:JNLP4-connect connection from 10.233.70.67/10.233.70.67:35132] will be ignored (a typical symptom is the Git executable not being run inside a designated container) [Pipeline] { Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@53d6545c; decorates RemoteLauncher[hudson.remoting.Channel@4fdded7d:JNLP4-connect connection from 10.233.66.114/10.233.66.114:55484] will be ignored (a typical symptom is the Git executable not being run inside a designated container) [Pipeline] { Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Uninstalling urllib3-1.26.16: Successfully uninstalled urllib3-1.26.16 Found existing installation: certifi 2023.5.7 Uninstalling certifi-2023.5.7: Successfully uninstalled certifi-2023.5.7 Found existing installation: idna 3.4 Uninstalling idna-3.4: Successfully uninstalled idna-3.4 Found existing installation: requests 2.27.1 Uninstalling requests-2.27.1: Successfully uninstalled requests-2.27.1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] cache Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Successfully installed certifi-2024.2.2 idna-3.7 requests-2.26.0 urllib3-1.26.18 start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 The 1 times to try to start tidb cluster... > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@fcad02d; decorates RemoteLauncher[hudson.remoting.Channel@4710483d:JNLP4-connect connection from 10.233.69.64/10.233.69.64:60030] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 check diff successfully start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 wait process cdc.test exit for 1-th time... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... Verifying downstream PD is started... wait process cdc.test exit for 2-th time... Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) wait process cdc.test exit for 3-th time... > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:47:54 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/http_api Starting Upstream PD... Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... 2024-05-06 19:47:55.264793 W | etcdserver: could not get cluster response from http://127.0.0.1:2680: Get "http://127.0.0.1:2680/members": dial tcp 127.0.0.1:2680: connect: connection refused 2024-05-06 19:47:55.264945 W | etcdserver: could not get cluster response from http://127.0.0.1:2780: Get "http://127.0.0.1:2780/members": dial tcp 127.0.0.1:2780: connect: connection refused > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36edd7400004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h, pid:1284, start at 2024-05-06 19:48:02.131832517 +0800 CST m=+5.376215138 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:50:02.140 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:48:02.128 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:38:02.128 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36edefdc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl, pid:1277, start at 2024-05-06 19:48:03.706936391 +0800 CST m=+5.460761006 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:50:03.715 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:48:03.703 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:38:03.703 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36edd7400004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h, pid:1284, start at 2024-05-06 19:48:02.131832517 +0800 CST m=+5.376215138 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:50:02.140 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:48:02.128 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:38:02.128 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36edd7f00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h, pid:1366, start at 2024-05-06 19:48:02.207839528 +0800 CST m=+5.386384526 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:50:02.214 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:48:02.172 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:38:02.172 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36edd8bc000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb, pid:1181, start at 2024-05-06 19:48:02.235464717 +0800 CST m=+5.273995016 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:50:02.242 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:48:02.223 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:38:02.223 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36edd8bc000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb, pid:1181, start at 2024-05-06 19:48:02.235464717 +0800 CST m=+5.273995016 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:50:02.242 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:48:02.223 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:38:02.223 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36edd96c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb, pid:1259, start at 2024-05-06 19:48:02.311158918 +0800 CST m=+5.302343814 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:50:02.318 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:48:02.316 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:38:02.316 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36edefdc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl, pid:1277, start at 2024-05-06 19:48:03.706936391 +0800 CST m=+5.460761006 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:50:03.715 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:48:03.703 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:38:03.703 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36edefec0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl, pid:1357, start at 2024-05-06 19:48:03.750954354 +0800 CST m=+5.423175686 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:50:03.758 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:48:03.757 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:38:03.757 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36ee07880012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:1322, start at 2024-05-06 19:48:05.242292013 +0800 CST m=+5.349011293 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:50:05.249 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:48:05.218 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:38:05.218 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36ee07880012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:1322, start at 2024-05-06 19:48:05.242292013 +0800 CST m=+5.349011293 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:50:05.249 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:48:05.218 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:38:05.218 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36ee07f40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:1411, start at 2024-05-06 19:48:05.247384268 +0800 CST m=+5.295645308 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:50:05.254 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:48:05.245 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:38:05.245 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2616.out cli tso query --pd=http://127.0.0.1:2379 TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_api/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/http_api/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/http_api/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/capture_suicide_while_balance_table Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2620.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575934329618433 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449575934329618433 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:48:08 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.26502652.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2612.out cli tso query --pd=http://127.0.0.1:2379 [Mon May 6 19:48:09 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.27232725.out server --log-file /tmp/tidb_cdc_test/http_api/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449575934380998657 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449575934380998657 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:48:09 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.26552657.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... + set +x + tso='449575934742495233 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449575934742495233 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "recordcount"="10" "updateproportion"="0" "requestdistribution"="uniform" "insertproportion"="0" "threadcount"="2" "scanproportion"="0" "readproportion"="0" "mysql.user"="root" "mysql.host"="127.0.0.1" "mysql.port"="4000" "operationcount"="0" "mysql.db"="sink_retry" "dotransactions"="false" "readallfields"="true" "workload"="core" ********************************************** Run finished, takes 8.617641ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2174.5, Avg(us): 1647, Min(us): 864, Max(us): 3990, 95th(us): 4000, 99th(us): 4000 [Mon May 6 19:48:10 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.26712673.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:48:11 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e668f7a3-29e2-4237-b80d-090d8c105bbb {"id":"e668f7a3-29e2-4237-b80d-090d8c105bbb","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996089} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb4cc73b e668f7a3-29e2-4237-b80d-090d8c105bbb /tidb/cdc/default/default/upstream/7365852043342538324 {"id":7365852043342538324,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e668f7a3-29e2-4237-b80d-090d8c105bbb {"id":"e668f7a3-29e2-4237-b80d-090d8c105bbb","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996089} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb4cc73b e668f7a3-29e2-4237-b80d-090d8c105bbb /tidb/cdc/default/default/upstream/7365852043342538324 {"id":7365852043342538324,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e668f7a3-29e2-4237-b80d-090d8c105bbb {"id":"e668f7a3-29e2-4237-b80d-090d8c105bbb","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996089} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb4cc73b e668f7a3-29e2-4237-b80d-090d8c105bbb /tidb/cdc/default/default/upstream/7365852043342538324 {"id":7365852043342538324,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2708.out cli changefeed create --start-ts=449575934329618433 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:48:12 GMT < Content-Length: 871 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7 {"id":"8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996089} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb5afb60 8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7 /tidb/cdc/default/default/upstream/7365852062317338733 {"id":7365852062317338733,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7 {"id":"8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996089} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb5afb60 8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7 /tidb/cdc/default/default/upstream/7365852062317338733 {"id":7365852062317338733,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7 {"id":"8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996089} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb5afb60 8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7 /tidb/cdc/default/default/upstream/7365852062317338733 {"id":7365852062317338733,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:48:12 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/555336a8-b1f5-4ae3-ba20-31e7910e6137 {"id":"555336a8-b1f5-4ae3-ba20-31e7910e6137","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996089} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb4dbb3b 555336a8-b1f5-4ae3-ba20-31e7910e6137 /tidb/cdc/default/default/upstream/7365852049085299392 {"id":7365852049085299392,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/555336a8-b1f5-4ae3-ba20-31e7910e6137 {"id":"555336a8-b1f5-4ae3-ba20-31e7910e6137","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996089} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb4dbb3b 555336a8-b1f5-4ae3-ba20-31e7910e6137 /tidb/cdc/default/default/upstream/7365852049085299392 {"id":7365852049085299392,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/555336a8-b1f5-4ae3-ba20-31e7910e6137 {"id":"555336a8-b1f5-4ae3-ba20-31e7910e6137","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996089} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb4dbb3b 555336a8-b1f5-4ae3-ba20-31e7910e6137 /tidb/cdc/default/default/upstream/7365852049085299392 {"id":7365852049085299392,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2720.out cli changefeed create --start-ts=449575934380998657 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 88a7b199-bb42-42b4-ac05-6b51c0f553c5 Info: {"upstream_id":7365852043342538324,"namespace":"default","id":"88a7b199-bb42-42b4-ac05-6b51c0f553c5","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:48:12.418823734+08:00","start_ts":449575934329618433,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575934329618433,"checkpoint_ts":449575934329618433,"checkpoint_time":"2024-05-06 19:48:07.378"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... Create changefeed successfully! ID: 388446aa-bfce-4080-8287-5b2c40fc7ba0 Info: {"upstream_id":7365852049085299392,"namespace":"default","id":"388446aa-bfce-4080-8287-5b2c40fc7ba0","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:48:12.585953216+08:00","start_ts":449575934380998657,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575934380998657,"checkpoint_ts":449575934380998657,"checkpoint_time":"2024-05-06 19:48:07.574"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "is-owner": true, run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:48:13 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e16b47c-307c-475e-8ebf-e59567902310 {"id":"0e16b47c-307c-475e-8ebf-e59567902310","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996091} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb528a50 0e16b47c-307c-475e-8ebf-e59567902310 /tidb/cdc/default/default/upstream/7365852046567954952 {"id":7365852046567954952,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e16b47c-307c-475e-8ebf-e59567902310 {"id":"0e16b47c-307c-475e-8ebf-e59567902310","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996091} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb528a50 0e16b47c-307c-475e-8ebf-e59567902310 /tidb/cdc/default/default/upstream/7365852046567954952 {"id":7365852046567954952,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e16b47c-307c-475e-8ebf-e59567902310 {"id":"0e16b47c-307c-475e-8ebf-e59567902310","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996091} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb528a50 0e16b47c-307c-475e-8ebf-e59567902310 /tidb/cdc/default/default/upstream/7365852046567954952 {"id":7365852046567954952,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2723.out cli changefeed create --start-ts=449575934742495233 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' + set +x + set +x Create changefeed successfully! ID: 23994825-1ed1-49f1-990b-91f64a998a62 Info: {"upstream_id":7365852046567954952,"namespace":"default","id":"23994825-1ed1-49f1-990b-91f64a998a62","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:48:14.156505114+08:00","start_ts":449575934742495233,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575934742495233,"checkpoint_ts":449575934742495233,"checkpoint_time":"2024-05-06 19:48:08.953"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) owner pid: 2728 owner id 8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7 default [Mon May 6 19:48:15 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.28682870.out server --log-file /tmp/tidb_cdc_test/http_api/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table foreign_key.finish_mark not exists for 1-th check, retry later + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL table foreign_key.finish_mark not exists for 2-th check, retry later * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:48:18 GMT < Content-Length: 1339 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27ba7849-9f72-445a-be11-3a2c411d582d {"id":"27ba7849-9f72-445a-be11-3a2c411d582d","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996095} /tidb/cdc/default/__cdc_meta__/capture/8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7 {"id":"8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996089} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb5afb60 8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb5afb98 27ba7849-9f72-445a-be11-3a2c411d582d /tidb/cdc/default/default/upstream/7365852062317338733 {"id":7365852062317338733,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27ba7849-9f72-445a-be11-3a2c411d582d {"id":"27ba7849-9f72-445a-be11-3a2c411d582d","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996095} /tidb/cdc/default/__cdc_meta__/capture/8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7 {"id":"8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996089} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb5afb60 8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb5afb98 27ba7849-9f72-445a-be11-3a2c411d582d /tidb/cdc/default/default/upstream/7365852062317338733 {"id":7365852062317338733,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27ba7849-9f72-445a-be11-3a2c411d582d {"id":"27ba7849-9f72-445a-be11-3a2c411d582d","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996095} /tidb/cdc/default/__cdc_meta__/capture/8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7 {"id":"8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996089} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb5afb60 8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb5afb98 27ba7849-9f72-445a-be11-3a2c411d582d /tidb/cdc/default/default/upstream/7365852062317338733 {"id":7365852062317338733,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "8c9753c9-ebdd-444d-b6b4-6ce10ca1f1c7 default" | grep id Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 24.92 secs (147191720 bytes/sec) [Pipeline] { [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36eed1140004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl, pid:3941, start at 2024-05-06 19:48:18.120139649 +0800 CST m=+5.336474363 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:50:18.127 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:48:18.117 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:38:18.117 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "id": "27ba7849-9f72-445a-be11-3a2c411d582d", run task successfully table foreign_key.finish_mark not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36eed1140004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl, pid:3941, start at 2024-05-06 19:48:18.120139649 +0800 CST m=+5.336474363 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:50:18.127 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:48:18.117 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:38:18.117 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36eed1b40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl, pid:4023, start at 2024-05-06 19:48:18.203330511 +0800 CST m=+5.368736885 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:50:18.210 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:48:18.207 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:38:18.207 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } capture_id: 27ba7849-9f72-445a-be11-3a2c411d582d pass test: check health pass test: get status table foreign_key.finish_mark not exists for 4-th check, retry later pass test: create changefeed [Mon May 6 19:48:23 CST 2024] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.53505352.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table foreign_key.finish_mark not exists for 5-th check, retry later table test.simple1 not exists for 1-th check, retry later table foreign_key.finish_mark not exists for 6-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:48:26 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d8d922b-0096-4d27-ba85-5ac600a526f5 {"id":"1d8d922b-0096-4d27-ba85-5ac600a526f5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996103} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb897933 1d8d922b-0096-4d27-ba85-5ac600a526f5 /tidb/cdc/default/default/upstream/7365852112192568876 {"id":7365852112192568876,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d8d922b-0096-4d27-ba85-5ac600a526f5 {"id":"1d8d922b-0096-4d27-ba85-5ac600a526f5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996103} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb897933 1d8d922b-0096-4d27-ba85-5ac600a526f5 /tidb/cdc/default/default/upstream/7365852112192568876 {"id":7365852112192568876,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d8d922b-0096-4d27-ba85-5ac600a526f5 {"id":"1d8d922b-0096-4d27-ba85-5ac600a526f5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996103} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb897933 1d8d922b-0096-4d27-ba85-5ac600a526f5 /tidb/cdc/default/default/upstream/7365852112192568876 {"id":7365852112192568876,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:48:26 CST 2024] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=1*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.54025404.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.simple1 not exists for 2-th check, retry later table foreign_key.finish_mark not exists for 7-th check, retry later table row_format.finish_mark not exists for 1-th check, retry later table test.simple1 exists pass test: list changefeed + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:48:29 GMT < Content-Length: 1295 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0db8cb6d-ad1d-4ca7-9133-65423bfb2dcd {"id":"0db8cb6d-ad1d-4ca7-9133-65423bfb2dcd","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996106} /tidb/cdc/default/__cdc_meta__/capture/1d8d922b-0096-4d27-ba85-5ac600a526f5 {"id":"1d8d922b-0096-4d27-ba85-5ac600a526f5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996103} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb897933 1d8d922b-0096-4d27-ba85-5ac600a526f5 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb897968 0db8cb6d-ad1d-4ca7-9133-65423bfb2dcd /tidb/cdc/default/default/upstream/7365852112192568876 {"id":7365852112192568876,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0db8cb6d-ad1d-4ca7-9133-65423bfb2dcd {"id":"0db8cb6d-ad1d-4ca7-9133-65423bfb2dcd","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996106} /tidb/cdc/default/__cdc_meta__/capture/1d8d922b-0096-4d27-ba85-5ac600a526f5 {"id":"1d8d922b-0096-4d27-ba85-5ac600a526f5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996103} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb897933 1d8d922b-0096-4d27-ba85-5ac600a526f5 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb897968 0db8cb6d-ad1d-4ca7-9133-65423bfb2dcd /tidb/cdc/default/default/upstream/7365852112192568876 {"id":7365852112192568876,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0db8cb6d-ad1d-4ca7-9133-65423bfb2dcd {"id":"0db8cb6d-ad1d-4ca7-9133-65423bfb2dcd","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996106} /tidb/cdc/default/__cdc_meta__/capture/1d8d922b-0096-4d27-ba85-5ac600a526f5 {"id":"1d8d922b-0096-4d27-ba85-5ac600a526f5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996103} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb897933 1d8d922b-0096-4d27-ba85-5ac600a526f5 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb897968 0db8cb6d-ad1d-4ca7-9133-65423bfb2dcd /tidb/cdc/default/default/upstream/7365852112192568876 {"id":7365852112192568876,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x pass test: get changefeed table capture_suicide_while_balance_table.t1 not exists for 1-th check, retry later pass test: pause changefeed table foreign_key.finish_mark not exists for 8-th check, retry later pass test: update changefeed wait process 2728 exit for 1-th time... table row_format.finish_mark not exists for 2-th check, retry later wait process 2728 exit for 2-th time... table capture_suicide_while_balance_table.t1 exists table capture_suicide_while_balance_table.t2 not exists for 1-th check, retry later table foreign_key.finish_mark not exists for 9-th check, retry later wait process 2728 exit for 3-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (2728) - No such process wait process 2728 exit for 4-th time... process 2728 already exit cdc.test cli capture list --server http://127.0.0.1:8301 2>&1 |grep 27ba7849-9f72-445a-be11-3a2c411d582d -A1 | grep '"is-owner": true' table row_format.finish_mark not exists for 3-th check, retry later table capture_suicide_while_balance_table.t2 exists table capture_suicide_while_balance_table.t3 not exists for 1-th check, retry later table foreign_key.finish_mark not exists for 10-th check, retry later "is-owner": true, run task successfully [Mon May 6 19:48:34 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.31443146.out server --log-file /tmp/tidb_cdc_test/http_api/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table row_format.finish_mark not exists for 4-th check, retry later table foreign_key.finish_mark not exists for 11-th check, retry later table capture_suicide_while_balance_table.t3 exists table capture_suicide_while_balance_table.t4 not exists for 1-th check, retry later table row_format.finish_mark not exists for 5-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:48:37 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/changefeed-test1 {UpstreamID:7365852062317338733 Namespace:default ID:changefeed-test1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:48:21.713162644 +0800 CST StartTs:449575938062549002 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002613200 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449575938075394059} {CheckpointTs:449575942086459396 MinTableBarrierTs:449575942086459396 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575942086459399, checkpointTs: 449575942086459396, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575942086459399, checkpointTs: 449575942086459396, state: Replicating changefeedID: default/changefeed-test3 {UpstreamID:7365852062317338733 Namespace:default ID:changefeed-test3 SinkURI:blackhole:// CreateTime:2024-05-06 19:48:22.012505309 +0800 CST StartTs:449575938141192197 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0036355f0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449575938154037261} {CheckpointTs:449575942086459396 MinTableBarrierTs:449575942086459396 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575942086459399, checkpointTs: 449575942086459396, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575942086459399, checkpointTs: 449575942086459396, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27ba7849-9f72-445a-be11-3a2c411d582d {"id":"27ba7849-9f72-445a-be11-3a2c411d582d","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996095} /tidb/cdc/default/__cdc_meta__/capture/d6d57f2e-c17f-4ced-b8ee-c90b1f371e3d {"id":"d6d57f2e-c17f-4ced-b8ee-c90b1f371e3d","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996114} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb5afb98 27ba7849-9f72-445a-be11-3a2c411d582d /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb5afc3c d6d57f2e-c17f-4ced-b8ee-c90b1f371e3d /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7365852062317338733,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:48:21.713162644+08:00","start-ts":449575938062549002,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575938075394059} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7365852062317338733,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-05-06T19:48:21.862801936+08:00","start-ts":449575938101870602,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"stopped","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575940251451401} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7365852062317338733,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-05-06T19:48:22.012505309+08:00","start-ts":449575938141192197,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575938154037261} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":449575942086459396,"min-table-barrier-ts":449575942086459396,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":449575938298216482,"min-table-barrier-ts":449575940251451395,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":449575942086459396,"min-table-barrier-ts":449575942086459396,"admin-job-type":0} /tidb/cdc/default/default/task/position/27ba7849-9f72-445a-be11-3a2c411d582d/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/27ba7849-9f72-445a-be11-3a2c411d582d/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d6d57f2e-c17f-4ced-b8ee-c90b1f371e3d/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d6d57f2e-c17f-4ced-b8ee-c90b1f371e3d/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365852062317338733 {"id":7365852062317338733,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/changefeed-test1 {UpstreamID:7365852062317338733 Namespace:default ID:changefeed-test1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:48:21.713162644 +0800 CST StartTs:449575938062549002 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002613200 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449575938075394059} {CheckpointTs:449575942086459396 MinTableBarrierTs:449575942086459396 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575942086459399, checkpointTs: 449575942086459396, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575942086459399, checkpointTs: 449575942086459396, state: Replicating changefeedID: default/changefeed-test3 {UpstreamID:7365852062317338733 Namespace:default ID:changefeed-test3 SinkURI:blackhole:// CreateTime:2024-05-06 19:48:22.012505309 +0800 CST StartTs:449575938141192197 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0036355f0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449575938154037261} {CheckpointTs:449575942086459396 MinTableBarrierTs:449575942086459396 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575942086459399, checkpointTs: 449575942086459396, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575942086459399, checkpointTs: 449575942086459396, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27ba7849-9f72-445a-be11-3a2c411d582d {"id":"27ba7849-9f72-445a-be11-3a2c411d582d","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996095} /tidb/cdc/default/__cdc_meta__/capture/d6d57f2e-c17f-4ced-b8ee-c90b1f371e3d {"id":"d6d57f2e-c17f-4ced-b8ee-c90b1f371e3d","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996114} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb5afb98 27ba7849-9f72-445a-be11-3a2c411d582d /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb5afc3c d6d57f2e-c17f-4ced-b8ee-c90b1f371e3d /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7365852062317338733,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:48:21.713162644+08:00","start-ts":449575938062549002,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575938075394059} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7365852062317338733,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-05-06T19:48:21.862801936+08:00","start-ts":449575938101870602,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"stopped","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575940251451401} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7365852062317338733,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-05-06T19:48:22.012505309+08:00","start-ts":449575938141192197,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575938154037261} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":449575942086459396,"min-table-barrier-ts":449575942086459396,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":449575938298216482,"min-table-barrier-ts":449575940251451395,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":449575942086459396,"min-table-barrier-ts":449575942086459396,"admin-job-type":0} /tidb/cdc/default/default/task/position/27ba7849-9f72-445a-be11-3a2c411d582d/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/27ba7849-9f72-445a-be11-3a2c411d582d/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d6d57f2e-c17f-4ced-b8ee-c90b1f371e3d/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d6d57f2e-c17f-4ced-b8ee-c90b1f371e3d/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365852062317338733 {"id":7365852062317338733,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/changefeed-test1 {UpstreamID:7365852062317338733 Namespace:default ID:changefeed-test1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:48:21.713162644 +0800 CST StartTs:449575938062549002 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002613200 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449575938075394059} {CheckpointTs:449575942086459396 MinTableBarrierTs:449575942086459396 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575942086459399, checkpointTs: 449575942086459396, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575942086459399, checkpointTs: 449575942086459396, state: Replicating changefeedID: default/changefeed-test3 {UpstreamID:7365852062317338733 Namespace:default ID:changefeed-test3 SinkURI:blackhole:// CreateTime:2024-05-06 19:48:22.012505309 +0800 CST StartTs:449575938141192197 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0036355f0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449575938154037261} {CheckpointTs:449575942086459396 MinTableBarrierTs:449575942086459396 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575942086459399, checkpointTs: 449575942086459396, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575942086459399, checkpointTs: 449575942086459396, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27ba7849-9f72-445a-be11-3a2c411d582d {"id":"27ba7849-9f72-445a-be11-3a2c411d582d","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996095} /tidb/cdc/default/__cdc_meta__/capture/d6d57f2e-c17f-4ced-b8ee-c90b1f371e3d {"id":"d6d57f2e-c17f-4ced-b8ee-c90b1f371e3d","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996114} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb5afb98 27ba7849-9f72-445a-be11-3a2c411d582d /tidb/cdc/default/__cdc_meta__/owner/22318f4dbb5afc3c d6d57f2e-c17f-4ced-b8ee-c90b1f371e3d /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7365852062317338733,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:48:21.713162644+08:00","start-ts":449575938062549002,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575938075394059} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7365852062317338733,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-05-06T19:48:21.862801936+08:00","start-ts":449575938101870602,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"stopped","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575940251451401} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7365852062317338733,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-05-06T19:48:22.012505309+08:00","start-ts":449575938141192197,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575938154037261} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":449575942086459396,"min-table-barrier-ts":449575942086459396,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":449575938298216482,"min-table-barrier-ts":449575940251451395,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":449575942086459396,"min-table-barrier-ts":449575942086459396,"admin-job-type":0} /tidb/cdc/default/default/task/position/27ba7849-9f72-445a-be11-3a2c411d582d/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/27ba7849-9f72-445a-be11-3a2c411d582d/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d6d57f2e-c17f-4ced-b8ee-c90b1f371e3d/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d6d57f2e-c17f-4ced-b8ee-c90b1f371e3d/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365852062317338733 {"id":7365852062317338733,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"address": "127.0.0.1:8300"' table foreign_key.finish_mark not exists for 12-th check, retry later table capture_suicide_while_balance_table.t4 exists "address": "127.0.0.1:8300", run task successfully table row_format.finish_mark not exists for 6-th check, retry later pass test: list changefeed pass test: get changefeed pass test: pause changefeed table foreign_key.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 140 100 35 100 105 722 2168 --:--:-- --:--:-- --:--:-- 2187 { "status": true, "message": "" pass test: resume changefeed wait process cdc.test exit for 2-th time... pass test: rebalance table table row_format.finish_mark not exists for 7-th check, retry later pass test: move table wait process cdc.test exit for 3-th time... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 20.42 secs (179650569 bytes/sec) [Pipeline] { [Pipeline] cache cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:48:42 CST 2024] <<<<<< run test case foreign_key success! >>>>>> table row_format.finish_mark not exists for 8-th check, retry later }lease 22318f4dbb897968 revoked check diff successfully wait process cdc.test exit for 1-th time... pass test: get processors wait process cdc.test exit for 2-th time... pass test: list processors pass test: set log level wait process cdc.test exit for 3-th time... pass test: remove changefeed cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:48:45 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> table row_format.finish_mark exists check diff successfully pass test: resign owner wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:48:47 CST 2024] <<<<<< run test case row_format success! >>>>>> table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "dotransactions"="false" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "mysql.port"="4000" "workload"="core" "readallfields"="true" "operationcount"="0" "threadcount"="2" "updateproportion"="0" "scanproportion"="0" "recordcount"="10" "mysql.user"="root" "mysql.db"="sink_retry" "insertproportion"="0" "readproportion"="0" ********************************************** Run finished, takes 4.389202ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3237.2, Avg(us): 764, Min(us): 536, Max(us): 1745, 95th(us): 2000, 99th(us): 2000 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 6.71 secs (546723388 bytes/sec) [Pipeline] { [Pipeline] cache =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/syncpoint Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f1df8c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl, pid:6871, start at 2024-05-06 19:49:08.238882842 +0800 CST m=+5.303637350 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:08.248 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:08.245 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:08.245 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f1c4cc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h, pid:3817, start at 2024-05-06 19:49:06.505498687 +0800 CST m=+5.275333403 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:06.515 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:06.483 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:06.483 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f1c4cc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h, pid:3817, start at 2024-05-06 19:49:06.505498687 +0800 CST m=+5.275333403 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:06.515 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:06.483 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:06.483 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f1c4c8000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h, pid:3887, start at 2024-05-06 19:49:06.493277282 +0800 CST m=+5.201087761 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:06.503 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:06.482 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:06.482 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f1df8c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl, pid:6871, start at 2024-05-06 19:49:08.238882842 +0800 CST m=+5.303637350 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:08.248 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:08.245 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:08.245 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f1e1b8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl, pid:6950, start at 2024-05-06 19:49:08.349310992 +0800 CST m=+5.360080099 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:08.355 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:08.334 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:08.334 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/syncpoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/syncpoint/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5192.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f1f2ac0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb, pid:3901, start at 2024-05-06 19:49:09.434813558 +0800 CST m=+5.428847682 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:09.444 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:09.419 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:09.419 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f1f2ac0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb, pid:3901, start at 2024-05-06 19:49:09.434813558 +0800 CST m=+5.428847682 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:09.444 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:09.419 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:09.419 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f1f280000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb, pid:3981, start at 2024-05-06 19:49:09.420150576 +0800 CST m=+5.349174440 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:09.427 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:09.408 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:09.408 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='449575951239479297 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449575951239479297 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.8215.out cli tso query --pd=http://127.0.0.1:2379 [Mon May 6 19:49:14 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.52415243.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5236.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575951753543681 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449575951753543681 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:49:15 CST 2024] <<<<<< START cdc server in syncpoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/DDLSinkInitializeSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.82518253.out server --log-file /tmp/tidb_cdc_test/syncpoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/syncpoint/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 21.14 secs (173492751 bytes/sec) [Pipeline] { [Pipeline] cache + set +x + tso='449575951969812481 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449575951969812481 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:49:16 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.52735275.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 pass test: get tso wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:49:17 CST 2024] <<<<<< run test case http_api success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:49:17 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/954a7344-4cd4-4e8b-abf2-10ab9e44db11 {"id":"954a7344-4cd4-4e8b-abf2-10ab9e44db11","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996154} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbc44d854 954a7344-4cd4-4e8b-abf2-10ab9e44db11 /tidb/cdc/default/default/upstream/7365852325709061779 {"id":7365852325709061779,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/954a7344-4cd4-4e8b-abf2-10ab9e44db11 {"id":"954a7344-4cd4-4e8b-abf2-10ab9e44db11","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996154} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbc44d854 954a7344-4cd4-4e8b-abf2-10ab9e44db11 /tidb/cdc/default/default/upstream/7365852325709061779 {"id":7365852325709061779,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/954a7344-4cd4-4e8b-abf2-10ab9e44db11 {"id":"954a7344-4cd4-4e8b-abf2-10ab9e44db11","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996154} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbc44d854 954a7344-4cd4-4e8b-abf2-10ab9e44db11 /tidb/cdc/default/default/upstream/7365852325709061779 {"id":7365852325709061779,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5302.out cli changefeed create --start-ts=449575951239479297 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: e1725191-8ace-42b9-8c4a-89fa187c2bff Info: {"upstream_id":7365852325709061779,"namespace":"default","id":"e1725191-8ace-42b9-8c4a-89fa187c2bff","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:49:17.967124553+08:00","start_ts":449575951239479297,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575951239479297,"checkpoint_ts":449575951239479297,"checkpoint_time":"2024-05-06 19:49:11.884"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:49:18 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1b7d7253-355b-4438-a5e8-e2bef4d85a69 {"id":"1b7d7253-355b-4438-a5e8-e2bef4d85a69","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996155} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbc4f6f42 1b7d7253-355b-4438-a5e8-e2bef4d85a69 /tidb/cdc/default/default/upstream/7365852332158580018 {"id":7365852332158580018,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1b7d7253-355b-4438-a5e8-e2bef4d85a69 {"id":"1b7d7253-355b-4438-a5e8-e2bef4d85a69","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996155} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbc4f6f42 1b7d7253-355b-4438-a5e8-e2bef4d85a69 /tidb/cdc/default/default/upstream/7365852332158580018 {"id":7365852332158580018,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1b7d7253-355b-4438-a5e8-e2bef4d85a69 {"id":"1b7d7253-355b-4438-a5e8-e2bef4d85a69","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996155} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbc4f6f42 1b7d7253-355b-4438-a5e8-e2bef4d85a69 /tidb/cdc/default/default/upstream/7365852332158580018 {"id":7365852332158580018,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.8313.out cli changefeed create --start-ts=449575951753543681 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/syncpoint/conf/changefeed.toml Create changefeed successfully! ID: 16f77e71-6e6b-4501-b07c-2ddaba8b3fe6 Info: {"upstream_id":7365852332158580018,"namespace":"default","id":"16f77e71-6e6b-4501-b07c-2ddaba8b3fe6","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:49:18.877296593+08:00","start_ts":449575951753543681,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575951753543681,"checkpoint_ts":449575951753543681,"checkpoint_time":"2024-05-06 19:49:13.845"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:49:19 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/20d6380b-34fa-4213-ac8e-e00cd6dbef66 {"id":"20d6380b-34fa-4213-ac8e-e00cd6dbef66","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996156} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbc57873b 20d6380b-34fa-4213-ac8e-e00cd6dbef66 /tidb/cdc/default/default/upstream/7365852336340006376 {"id":7365852336340006376,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/20d6380b-34fa-4213-ac8e-e00cd6dbef66 {"id":"20d6380b-34fa-4213-ac8e-e00cd6dbef66","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996156} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbc57873b 20d6380b-34fa-4213-ac8e-e00cd6dbef66 /tidb/cdc/default/default/upstream/7365852336340006376 {"id":7365852336340006376,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/20d6380b-34fa-4213-ac8e-e00cd6dbef66 {"id":"20d6380b-34fa-4213-ac8e-e00cd6dbef66","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996156} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbc57873b 20d6380b-34fa-4213-ac8e-e00cd6dbef66 /tidb/cdc/default/default/upstream/7365852336340006376 {"id":7365852336340006376,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5335.out cli changefeed create --start-ts=449575951969812481 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 12f19f37-23e1-43d6-b1b9-c156c450ba61 Info: {"upstream_id":7365852336340006376,"namespace":"default","id":"12f19f37-23e1-43d6-b1b9-c156c450ba61","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:49:19.715715196+08:00","start_ts":449575951969812481,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575951969812481,"checkpoint_ts":449575951969812481,"checkpoint_time":"2024-05-06 19:49:14.670"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:49:18 CST 2024] <<<<<< run test case sink_retry success! >>>>>> + set +x ***************** properties ***************** "insertproportion"="0" "mysql.db"="testSync" "threadcount"="2" "mysql.host"="127.0.0.1" "workload"="core" "readallfields"="true" "updateproportion"="0" "operationcount"="0" "dotransactions"="false" "scanproportion"="0" "mysql.port"="4000" "requestdistribution"="uniform" "readproportion"="0" "mysql.user"="root" "recordcount"="10" ********************************************** Run finished, takes 8.924813ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1948.8, Avg(us): 1719, Min(us): 1084, Max(us): 3744, 95th(us): 4000, 99th(us): 4000 + set +x table drop_tables.c not exists for 1-th check, retry later table drop_tables.c not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table drop_tables.c not exists for 3-th check, retry later table drop_tables.c not exists for 4-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_api_tls/run.sh using Sink-Type: mysql... <<================= WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Requirement already up-to-date: requests==2.26.0 in /usr/local/lib/python3.6/site-packages Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: idna<4,>=2.5; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: certifi>=2017.4.17 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) The 1 times to try to start tidb cluster... table drop_tables.c exists check diff successfully start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:49:35 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 17.21 secs (213134177 bytes/sec) [Pipeline] { [Pipeline] cache Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f416e80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl, pid:3933, start at 2024-05-06 19:49:44.512814944 +0800 CST m=+5.215527110 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:44.522 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:44.506 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:44.506 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f421d80017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:4192, start at 2024-05-06 19:49:45.242391457 +0800 CST m=+5.191969608 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:45.248 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:45.206 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:45.206 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f421d80017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:4192, start at 2024-05-06 19:49:45.242391457 +0800 CST m=+5.191969608 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:45.248 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:45.206 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:45.206 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f4240c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:4272, start at 2024-05-06 19:49:45.367750829 +0800 CST m=+5.236304700 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:45.374 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:45.347 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:45.347 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api_tls/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api_tls/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/http_api_tls/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_api_tls/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/http_api_tls/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f416e80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl, pid:3933, start at 2024-05-06 19:49:44.512814944 +0800 CST m=+5.215527110 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:44.522 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:44.506 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:44.506 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f4187c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl, pid:4018, start at 2024-05-06 19:49:44.61644267 +0800 CST m=+5.266363399 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:44.622 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:44.607 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:44.607 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting TLS PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5284.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... + set +x + tso='449575961219825665 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449575961219825665 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "readallfields"="true" "mysql.user"="root" "readproportion"="0" "requestdistribution"="uniform" "scanproportion"="0" "mysql.port"="4000" "dotransactions"="false" "operationcount"="0" "mysql.db"="changefeed_error" "recordcount"="20" "workload"="core" "threadcount"="4" "updateproportion"="0" "insertproportion"="0" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 10.127596ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3159.6, Avg(us): 1741, Min(us): 888, Max(us): 5585, 95th(us): 6000, 99th(us): 6000 [Mon May 6 19:49:51 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.53435345.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting TLS TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting TLS TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:49:54 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/991a998c-12c6-4320-b1ed-24b1e905ea4c {"id":"991a998c-12c6-4320-b1ed-24b1e905ea4c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996191} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbcd98f4f 991a998c-12c6-4320-b1ed-24b1e905ea4c /tidb/cdc/default/default/upstream/7365852492112483417 {"id":7365852492112483417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/991a998c-12c6-4320-b1ed-24b1e905ea4c {"id":"991a998c-12c6-4320-b1ed-24b1e905ea4c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996191} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbcd98f4f 991a998c-12c6-4320-b1ed-24b1e905ea4c /tidb/cdc/default/default/upstream/7365852492112483417 {"id":7365852492112483417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/991a998c-12c6-4320-b1ed-24b1e905ea4c {"id":"991a998c-12c6-4320-b1ed-24b1e905ea4c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996191} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbcd98f4f 991a998c-12c6-4320-b1ed-24b1e905ea4c /tidb/cdc/default/default/upstream/7365852492112483417 {"id":7365852492112483417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5401.out cli changefeed create --start-ts=449575961219825665 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7365852492112483417,"namespace":"default","id":"changefeed-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:49:55.137966041+08:00","start_ts":449575961219825665,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575961219825665,"checkpoint_ts":449575961219825665,"checkpoint_time":"2024-05-06 19:49:49.956"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 15.86 secs (231246446 bytes/sec) [Pipeline] { [Pipeline] cache + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449575961219825665, "checkpoint_time": "2024-05-06 19:49:49.956", "error": { "time": "2024-05-06T19:49:55.248828759+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449575961219825365 is earlier than or equal to GC safepoint at 449575961219825665" } }' + echo '{ "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449575961219825665, "checkpoint_time": "2024-05-06 19:49:49.956", "error": { "time": "2024-05-06T19:49:55.248828759+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449575961219825365 is earlier than or equal to GC safepoint at 449575961219825665" } }' { "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449575961219825665, "checkpoint_time": "2024-05-06 19:49:49.956", "error": { "time": "2024-05-06T19:49:55.248828759+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449575961219825365 is earlier than or equal to GC safepoint at 449575961219825665" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365852492112483417, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449575961219825665, '"checkpoint_time":' '"2024-05-06' '19:49:49.956",' '"error":' '{' '"time":' '"2024-05-06T19:49:55.248828759+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449575961219825365 is earlier than or equal to GC safepoint at '449575961219825665"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7365852492112483417, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449575961219825665, '"checkpoint_time":' '"2024-05-06' '19:49:49.956",' '"error":' '{' '"time":' '"2024-05-06T19:49:55.248828759+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449575961219825365 is earlier than or equal to GC safepoint at '449575961219825665"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449575961219825365 is earlier than or equal to GC safepoint at 449575961219825665' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449575961219825365 is earlier than or equal to GC safepoint at 449575961219825665 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5485.out cli changefeed resume -c changefeed-error ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS coverage: 2.3% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table changefeed_error.usertable not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f4f09c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:5934, start at 2024-05-06 19:49:58.445348598 +0800 CST m=+5.152661011 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:58.451 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:58.439 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:58.439 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Mon May 6 19:49:59 CST 2024] <<<<<< START cdc server in http_api_tls case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls.59945996.out server --log-file /tmp/tidb_cdc_test/http_api_tls/cdc_http_api_tls_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls/cdc_data_http_api_tls_tls1 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f507000003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb, pid:6456, start at 2024-05-06 19:49:59.875729746 +0800 CST m=+5.336866654 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:59.882 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:59.872 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:59.872 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_error.usertable exists check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f507000003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb, pid:6456, start at 2024-05-06 19:49:59.875729746 +0800 CST m=+5.336866654 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:59.882 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:59.872 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:59.872 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f507b00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb, pid:6541, start at 2024-05-06 19:49:59.944887326 +0800 CST m=+5.358650451 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:51:59.953 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:49:59.916 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:39:59.916 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:50:02 GMT < Content-Length: 1281 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/62768d7a-100a-446d-a4c6-cfff7d4a2043 {"id":"62768d7a-100a-446d-a4c6-cfff7d4a2043","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996199} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dbd153b32 62768d7a-100a-446d-a4c6-cfff7d4a2043 /tidb/cdc/default/default/upstream/7365852546830488886 {"id":7365852546830488886,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/62768d7a-100a-446d-a4c6-cfff7d4a2043 {"id":"62768d7a-100a-446d-a4c6-cfff7d4a2043","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996199} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dbd153b32 62768d7a-100a-446d-a4c6-cfff7d4a2043 /tidb/cdc/default/default/upstream/7365852546830488886 {"id":7365852546830488886,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/62768d7a-100a-446d-a4c6-cfff7d4a2043 {"id":"62768d7a-100a-446d-a4c6-cfff7d4a2043","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996199} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dbd153b32 62768d7a-100a-446d-a4c6-cfff7d4a2043 /tidb/cdc/default/default/upstream/7365852546830488886 {"id":7365852546830488886,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x check diff successfully ***************** properties ***************** "readallfields"="true" "operationcount"="0" "mysql.host"="127.0.0.1" "updateproportion"="0" "mysql.port"="4000" "mysql.db"="changefeed_error" "dotransactions"="false" "mysql.user"="root" "recordcount"="20" "scanproportion"="0" "threadcount"="4" "insertproportion"="0" "readproportion"="0" "requestdistribution"="uniform" "workload"="core" ********************************************** Run finished, takes 4.702699ms INSERT - Takes(s): 0.0, Count: 20, OPS: 6322.3, Avg(us): 799, Min(us): 485, Max(us): 1847, 95th(us): 2000, 99th(us): 2000 check diff successfully {"id":"991a998c-12c6-4320-b1ed-24b1e905ea4c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996191} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/991a998c-12c6-4320-b1ed-24b1e905ea4c + [[ /tidb/cdc/default/__cdc_meta__/capture/991a998c-12c6-4320-b1ed-24b1e905ea4c =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/991a998c-12c6-4320-b1ed-24b1e905ea4c' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/991a998c-12c6-4320-b1ed-24b1e905ea4c + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later [Mon May 6 19:50:04 CST 2024] <<<<<< START cdc server in http_api_tls case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates ']' + GO_FAILPOINTS= + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls.60556057.out server --log-file /tmp/tidb_cdc_test/http_api_tls/cdc_http_api_tls_tls2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls/cdc_data_http_api_tls_tls2 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8301 --pd https://127.0.0.1:2579 + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 19:50:04 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.77927794.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/991a998c-12c6-4320-b1ed-24b1e905ea4c + [[ /tidb/cdc/default/__cdc_meta__/capture/991a998c-12c6-4320-b1ed-24b1e905ea4c =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/991a998c-12c6-4320-b1ed-24b1e905ea4c' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/991a998c-12c6-4320-b1ed-24b1e905ea4c + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:50:07 GMT < Content-Length: 1749 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e853ca9-c4a3-4028-aeaf-39250112c2c7 {"id":"3e853ca9-c4a3-4028-aeaf-39250112c2c7","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996205} /tidb/cdc/default/__cdc_meta__/capture/62768d7a-100a-446d-a4c6-cfff7d4a2043 {"id":"62768d7a-100a-446d-a4c6-cfff7d4a2043","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996199} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dbd153b32 62768d7a-100a-446d-a4c6-cfff7d4a2043 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dbd153b71 3e853ca9-c4a3-4028-aeaf-39250112c2c7 /tidb/cdc/default/default/upstream/7365852546830488886 {"id":7365852546830488886,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e853ca9-c4a3-4028-aeaf-39250112c2c7 {"id":"3e853ca9-c4a3-4028-aeaf-39250112c2c7","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996205} /tidb/cdc/default/__cdc_meta__/capture/62768d7a-100a-446d-a4c6-cfff7d4a2043 {"id":"62768d7a-100a-446d-a4c6-cfff7d4a2043","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996199} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dbd153b32 62768d7a-100a-446d-a4c6-cfff7d4a2043 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dbd153b71 3e853ca9-c4a3-4028-aeaf-39250112c2c7 /tidb/cdc/default/default/upstream/7365852546830488886 {"id":7365852546830488886,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e853ca9-c4a3-4028-aeaf-39250112c2c7 {"id":"3e853ca9-c4a3-4028-aeaf-39250112c2c7","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996205} /tidb/cdc/default/__cdc_meta__/capture/62768d7a-100a-446d-a4c6-cfff7d4a2043 {"id":"62768d7a-100a-446d-a4c6-cfff7d4a2043","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996199} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dbd153b32 62768d7a-100a-446d-a4c6-cfff7d4a2043 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dbd153b71 3e853ca9-c4a3-4028-aeaf-39250112c2c7 /tidb/cdc/default/default/upstream/7365852546830488886 {"id":7365852546830488886,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:50:07 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef61f726-b0ba-403c-8a2a-25af5ca3d1a3 {"id":"ef61f726-b0ba-403c-8a2a-25af5ca3d1a3","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996205} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd150138 ef61f726-b0ba-403c-8a2a-25af5ca3d1a3 /tidb/cdc/default/default/upstream/7365852548432472151 {"id":7365852548432472151,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef61f726-b0ba-403c-8a2a-25af5ca3d1a3 {"id":"ef61f726-b0ba-403c-8a2a-25af5ca3d1a3","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996205} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd150138 ef61f726-b0ba-403c-8a2a-25af5ca3d1a3 /tidb/cdc/default/default/upstream/7365852548432472151 {"id":7365852548432472151,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef61f726-b0ba-403c-8a2a-25af5ca3d1a3 {"id":"ef61f726-b0ba-403c-8a2a-25af5ca3d1a3","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996205} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd150138 ef61f726-b0ba-403c-8a2a-25af5ca3d1a3 /tidb/cdc/default/default/upstream/7365852548432472151 {"id":7365852548432472151,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table processor_stop_delay.t not exists for 1-th check, retry later pass test: check health check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/991a998c-12c6-4320-b1ed-24b1e905ea4c + [[ /tidb/cdc/default/__cdc_meta__/capture/991a998c-12c6-4320-b1ed-24b1e905ea4c =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/991a998c-12c6-4320-b1ed-24b1e905ea4c' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/991a998c-12c6-4320-b1ed-24b1e905ea4c + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later pass test: get status table processor_stop_delay.t exists check diff failed 1-th time, retry later pass test: create changefeed check_changefeed_state https://127.0.0.1:2579 changefeed-test1 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test1 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test1 -s + info='{ "upstream_id": 7365852546830488886, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 449575966550261769, "checkpoint_time": "2024-05-06 19:50:10.290", "error": null }' + echo '{ "upstream_id": 7365852546830488886, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 449575966550261769, "checkpoint_time": "2024-05-06 19:50:10.290", "error": null }' { "upstream_id": 7365852546830488886, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 449575966550261769, "checkpoint_time": "2024-05-06 19:50:10.290", "error": null } ++ echo '{' '"upstream_id":' 7365852546830488886, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 449575966550261769, '"checkpoint_time":' '"2024-05-06' '19:50:10.290",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365852546830488886, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 449575966550261769, '"checkpoint_time":' '"2024-05-06' '19:50:10.290",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test2 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test2 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test2 -s + info='{ "upstream_id": 7365852546830488886, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 449575966615797766, "checkpoint_time": "2024-05-06 19:50:10.540", "error": null }' + echo '{ "upstream_id": 7365852546830488886, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 449575966615797766, "checkpoint_time": "2024-05-06 19:50:10.540", "error": null }' { "upstream_id": 7365852546830488886, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 449575966615797766, "checkpoint_time": "2024-05-06 19:50:10.540", "error": null } ++ echo '{' '"upstream_id":' 7365852546830488886, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 449575966615797766, '"checkpoint_time":' '"2024-05-06' '19:50:10.540",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365852546830488886, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 449575966615797766, '"checkpoint_time":' '"2024-05-06' '19:50:10.540",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test3 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test3 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test3 -s + info='{ "upstream_id": 7365852546830488886, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 449575966654857224, "checkpoint_time": "2024-05-06 19:50:10.689", "error": null }' + echo '{ "upstream_id": 7365852546830488886, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 449575966654857224, "checkpoint_time": "2024-05-06 19:50:10.689", "error": null }' { "upstream_id": 7365852546830488886, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 449575966654857224, "checkpoint_time": "2024-05-06 19:50:10.689", "error": null } ++ echo '{' '"upstream_id":' 7365852546830488886, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 449575966654857224, '"checkpoint_time":' '"2024-05-06' '19:50:10.689",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365852546830488886, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 449575966654857224, '"checkpoint_time":' '"2024-05-06' '19:50:10.689",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test4 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test4 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test4 -s + info='{ "upstream_id": 7365852546830488886, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 449575966707286018, "checkpoint_time": "2024-05-06 19:50:10.889", "error": null }' + echo '{ "upstream_id": 7365852546830488886, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 449575966707286018, "checkpoint_time": "2024-05-06 19:50:10.889", "error": null }' { "upstream_id": 7365852546830488886, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 449575966707286018, "checkpoint_time": "2024-05-06 19:50:10.889", "error": null } ++ echo '{' '"upstream_id":' 7365852546830488886, '"namespace":' '"default",' '"id":' '"changefeed-test4",' '"state":' '"normal",' '"checkpoint_tso":' 449575966707286018, '"checkpoint_time":' '"2024-05-06' '19:50:10.889",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365852546830488886, '"namespace":' '"default",' '"id":' '"changefeed-test4",' '"state":' '"normal",' '"checkpoint_tso":' 449575966707286018, '"checkpoint_time":' '"2024-05-06' '19:50:10.889",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check diff successfully Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 15.90 secs (230739760 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh check diff failed 1-th time, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G08 Run cases: processor_err_chan changefeed_reconstruct multi_capture synced_status_with_redo PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=9cb25ae7-1e3b-45d6-88b3-2df3ecc7bf2e BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-3z0ms GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-3z0ms cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=caf2d0a2-2010-4ca1-a11f-9b057b403d23 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3n4n5 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-bltlv GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-bltlv cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3n4n5 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3n4n5 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G05 Run cases: charset_gbk ddl_manager multi_source PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=380a29fb-1d39-48cd-80b2-1dffc662779c BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-mzzb9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-gx9jn GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-gx9jn cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-mzzb9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-mzzb9 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh table test.simple1 exists pass test: list changefeed + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=9441c639-e047-4769-b4ee-e9e70fcb5b8b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-7312q GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84 cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-7312q GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=356349f1-e413-4f38-8ae9-c3e8ae20d232 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-qsh8g GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-qsh8g cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G09 Run cases: gc_safepoint changefeed_pause_resume cli_with_auth savepoint synced_status PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=14f66c62-d6d1-4e5d-a05a-cae157a1acad BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-l18q5 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-l18q5 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 19:50:15 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.57485750.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G10 Run cases: default_value simple cdc_server_tips event_filter sql_mode PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=db92d544-38ba-4a1c-84d0-4286159064fd BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-dk00k GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72 cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-dk00k GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: mysql... <<================= pass test: get changefeed start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:50:18 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4b135ce-f682-47d3-b8d2-12ef987d2417 {"id":"a4b135ce-f682-47d3-b8d2-12ef987d2417","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996215} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbcd9903d a4b135ce-f682-47d3-b8d2-12ef987d2417 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365852492112483417,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:49:55.137966041+08:00","start-ts":449575961219825665,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-06T19:50:16.018846105+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575968048676866} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449575964470411266,"min-table-barrier-ts":449575964470411266,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365852492112483417 {"id":7365852492112483417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4b135ce-f682-47d3-b8d2-12ef987d2417 {"id":"a4b135ce-f682-47d3-b8d2-12ef987d2417","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996215} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbcd9903d a4b135ce-f682-47d3-b8d2-12ef987d2417 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365852492112483417,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:49:55.137966041+08:00","start-ts":449575961219825665,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-06T19:50:16.018846105+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575968048676866} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449575964470411266,"min-table-barrier-ts":449575964470411266,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365852492112483417 {"id":7365852492112483417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4b135ce-f682-47d3-b8d2-12ef987d2417 {"id":"a4b135ce-f682-47d3-b8d2-12ef987d2417","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996215} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbcd9903d a4b135ce-f682-47d3-b8d2-12ef987d2417 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365852492112483417,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:49:55.137966041+08:00","start-ts":449575961219825665,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-06T19:50:16.018846105+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575968048676866} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449575964470411266,"min-table-barrier-ts":449575964470411266,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365852492112483417 {"id":7365852492112483417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449575964470411266, "checkpoint_time": "2024-05-06 19:50:02.356", "error": { "time": "2024-05-06T19:50:16.018846105+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449575964470411266, "checkpoint_time": "2024-05-06 19:50:02.356", "error": { "time": "2024-05-06T19:50:16.018846105+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449575964470411266, "checkpoint_time": "2024-05-06 19:50:02.356", "error": { "time": "2024-05-06T19:50:16.018846105+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7365852492112483417, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449575964470411266, '"checkpoint_time":' '"2024-05-06' '19:50:02.356",' '"error":' '{' '"time":' '"2024-05-06T19:50:16.018846105+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7365852492112483417, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449575964470411266, '"checkpoint_time":' '"2024-05-06' '19:50:02.356",' '"error":' '{' '"time":' '"2024-05-06T19:50:16.018846105+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message pass test: pause changefeed + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5845.out cli changefeed remove -c changefeed-error check diff failed 3-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... Changefeed remove successfully. ID: changefeed-error CheckpointTs: 449575964470411266 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_ddl Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release pass test: update changefeed Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... The 1 times to try to start tidb cluster... check diff failed 4-th time, retry later Verifying downstream PD is started... + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully wait process cdc.test exit for 1-th time... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 2-th time... pass test: resume changefeed Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 19:50:22 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.59445946.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 5-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { pass test: rebalance table Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: list processors ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully wait process cdc.test exit for 1-th time... [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:50:25 GMT < Content-Length: 895 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e9e0f125-32c2-4c73-b8b3-7a38b9d02869 {"id":"e9e0f125-32c2-4c73-b8b3-7a38b9d02869","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996223} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbcd9908b e9e0f125-32c2-4c73-b8b3-7a38b9d02869 /tidb/cdc/default/default/upstream/7365852492112483417 {"id":7365852492112483417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e9e0f125-32c2-4c73-b8b3-7a38b9d02869 {"id":"e9e0f125-32c2-4c73-b8b3-7a38b9d02869","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996223} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbcd9908b e9e0f125-32c2-4c73-b8b3-7a38b9d02869 /tidb/cdc/default/default/upstream/7365852492112483417 {"id":7365852492112483417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e9e0f125-32c2-4c73-b8b3-7a38b9d02869 {"id":"e9e0f125-32c2-4c73-b8b3-7a38b9d02869","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996223} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbcd9908b e9e0f125-32c2-4c73-b8b3-7a38b9d02869 /tidb/cdc/default/default/upstream/7365852492112483417 {"id":7365852492112483417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5999.out cli changefeed create --start-ts=449575961219825665 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-1 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7365852492112483417,"namespace":"default","id":"changefeed-error-1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:50:26.195529315+08:00","start_ts":449575961219825665,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575961219825665,"checkpoint_ts":449575961219825665,"checkpoint_time":"2024-05-06 19:49:49.956"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:50:26 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 253 100 253 0 0 2323 0 --:--:-- --:--:-- --:--:-- 2342 + info='{"state":"warning","resolved_ts":449575961599934482,"checkpoint_ts":449575961599934482,"last_warning":{"time":"2024-05-06T19:50:27.674609864+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":449575961599934482,"checkpoint_ts":449575961599934482,"last_warning":{"time":"2024-05-06T19:50:27.674609864+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":449575961599934482,"checkpoint_ts":449575961599934482,"last_warning":{"time":"2024-05-06T19:50:27.674609864+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ echo '{"state":"warning","resolved_ts":449575961599934482,"checkpoint_ts":449575961599934482,"last_warning":{"time":"2024-05-06T19:50:27.674609864+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ echo '{"state":"warning","resolved_ts":449575961599934482,"checkpoint_ts":449575961599934482,"last_warning":{"time":"2024-05-06T19:50:27.674609864+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .last_warning.message + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6048.out cli changefeed remove -c changefeed-error-1 [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { ***************** properties ***************** "readproportion"="0" "mysql.db"="testSync" "operationcount"="0" "insertproportion"="0" "requestdistribution"="uniform" "mysql.user"="root" "mysql.port"="4000" "scanproportion"="0" "readallfields"="true" "mysql.host"="127.0.0.1" "updateproportion"="0" "recordcount"="10" "dotransactions"="false" "workload"="core" "threadcount"="2" ********************************************** Run finished, takes 6.720236ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1910.2, Avg(us): 1083, Min(us): 540, Max(us): 2858, 95th(us): 3000, 99th(us): 3000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN pass test: get processor Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 449575961599934482 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] { ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] cache pass test: move table ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f6d21c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:1292, start at 2024-05-06 19:50:29.305201715 +0800 CST m=+5.405376700 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:29.315 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:29.305 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:29.305 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f6b8100019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:1180, start at 2024-05-06 19:50:27.623845825 +0800 CST m=+5.289935531 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:27.634 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:27.638 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:27.638 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f6b8100019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:1180, start at 2024-05-06 19:50:27.623845825 +0800 CST m=+5.289935531 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:27.634 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:27.638 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:27.638 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f6b980000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:1260, start at 2024-05-06 19:50:27.692477835 +0800 CST m=+5.306922734 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:27.701 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:27.680 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:27.680 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } pass test: set log level ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f6cd3c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-mzzb9, pid:1231, start at 2024-05-06 19:50:28.970080796 +0800 CST m=+5.295098974 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:28.978 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:28.943 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:28.943 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f6cd3c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-mzzb9, pid:1231, start at 2024-05-06 19:50:28.970080796 +0800 CST m=+5.295098974 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:28.978 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:28.943 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:28.943 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f6ced00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-mzzb9, pid:1312, start at 2024-05-06 19:50:29.0450989 +0800 CST m=+5.306159445 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:29.051 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:29.044 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:29.044 +0800 All versions after safe point can be accessed. (DO NOT EDIT) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 19:50:31 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.61136115.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f6d21c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:1292, start at 2024-05-06 19:50:29.305201715 +0800 CST m=+5.405376700 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:29.315 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:29.305 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:29.305 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f6d52c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:1377, start at 2024-05-06 19:50:29.465089883 +0800 CST m=+5.487016281 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:29.474 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:29.451 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:29.451 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } pass test: remove changefeed VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f6d8f00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3n4n5, pid:1242, start at 2024-05-06 19:50:29.714777168 +0800 CST m=+5.440024031 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:29.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:29.692 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:29.692 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f6d8f00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3n4n5, pid:1242, start at 2024-05-06 19:50:29.714777168 +0800 CST m=+5.440024031 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:29.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:29.692 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:29.692 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f6da7c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3n4n5, pid:1319, start at 2024-05-06 19:50:29.798659441 +0800 CST m=+5.470878852 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:29.806 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:29.791 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:29.791 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f6e8980004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:1178, start at 2024-05-06 19:50:30.696915881 +0800 CST m=+5.406714325 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:30.706 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:30.694 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:30.694 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f6e8980004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:1178, start at 2024-05-06 19:50:30.696915881 +0800 CST m=+5.406714325 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:30.706 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:30.694 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:30.694 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f6ebbc0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:1259, start at 2024-05-06 19:50:30.930474643 +0800 CST m=+5.557954258 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:30.937 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:30.895 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:30.895 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } pass test: resign owner [Mon May 6 19:50:34 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.26352637.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 19:50:34 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.26552657.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 19:50:34 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.26252627.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:50:34 GMT < Content-Length: 895 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c7376cb-d75a-4646-ace6-f12cb58ae775 {"id":"6c7376cb-d75a-4646-ace6-f12cb58ae775","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996232} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbcd99107 6c7376cb-d75a-4646-ace6-f12cb58ae775 /tidb/cdc/default/default/upstream/7365852492112483417 {"id":7365852492112483417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c7376cb-d75a-4646-ace6-f12cb58ae775 {"id":"6c7376cb-d75a-4646-ace6-f12cb58ae775","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996232} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbcd99107 6c7376cb-d75a-4646-ace6-f12cb58ae775 /tidb/cdc/default/default/upstream/7365852492112483417 {"id":7365852492112483417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c7376cb-d75a-4646-ace6-f12cb58ae775 {"id":"6c7376cb-d75a-4646-ace6-f12cb58ae775","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996232} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbcd99107 6c7376cb-d75a-4646-ace6-f12cb58ae775 /tidb/cdc/default/default/upstream/7365852492112483417 {"id":7365852492112483417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6171.out cli changefeed create --start-ts=449575961219825665 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-2 Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7365852492112483417,"namespace":"default","id":"changefeed-error-2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:50:35.311482084+08:00","start_ts":449575961219825665,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575961219825665,"checkpoint_ts":449575961219825665,"checkpoint_time":"2024-05-06 19:49:49.956"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f71dc40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:1246, start at 2024-05-06 19:50:34.112974393 +0800 CST m=+5.181280864 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:34.122 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:34.097 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:34.097 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f71dc40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:1246, start at 2024-05-06 19:50:34.112974393 +0800 CST m=+5.181280864 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:34.122 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:34.097 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:34.097 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f71e640013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:1327, start at 2024-05-06 19:50:34.163016134 +0800 CST m=+5.179952701 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:34.170 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:34.137 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:34.137 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/default_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/default_value/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/default_value/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } pass test: get tso VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f724140012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc, pid:1241, start at 2024-05-06 19:50:34.52918581 +0800 CST m=+5.288593686 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:34.537 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:34.501 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:34.501 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 19:50:35 CST 2024] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.26332635.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s pass test: verify table + info='{ "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449575961219825665, "checkpoint_time": "2024-05-06 19:49:49.956", "error": { "time": "2024-05-06T19:50:35.390789607+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575961219825665 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449575961219825665, "checkpoint_time": "2024-05-06 19:49:49.956", "error": { "time": "2024-05-06T19:50:35.390789607+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575961219825665 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449575961219825665, "checkpoint_time": "2024-05-06 19:49:49.956", "error": { "time": "2024-05-06T19:50:35.390789607+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575961219825665 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ echo '{' '"upstream_id":' 7365852492112483417, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449575961219825665, '"checkpoint_time":' '"2024-05-06' '19:49:49.956",' '"error":' '{' '"time":' '"2024-05-06T19:50:35.390789607+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575961219825665 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365852492112483417, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449575961219825665, '"checkpoint_time":' '"2024-05-06' '19:49:49.956",' '"error":' '{' '"time":' '"2024-05-06T19:50:35.390789607+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575961219825665 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575961219825665 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575961219825665 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6257.out cli changefeed remove -c changefeed-error-2 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:50:37 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3bbd10a5-3adb-4879-bf36-7d24df1fe9b4 {"id":"3bbd10a5-3adb-4879-bf36-7d24df1fe9b4","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996234} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd8f9833 3bbd10a5-3adb-4879-bf36-7d24df1fe9b4 /tidb/cdc/default/default/upstream/7365852681056034993 {"id":7365852681056034993,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3bbd10a5-3adb-4879-bf36-7d24df1fe9b4 {"id":"3bbd10a5-3adb-4879-bf36-7d24df1fe9b4","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996234} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd8f9833 3bbd10a5-3adb-4879-bf36-7d24df1fe9b4 /tidb/cdc/default/default/upstream/7365852681056034993 {"id":7365852681056034993,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3bbd10a5-3adb-4879-bf36-7d24df1fe9b4 {"id":"3bbd10a5-3adb-4879-bf36-7d24df1fe9b4","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996234} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd8f9833 3bbd10a5-3adb-4879-bf36-7d24df1fe9b4 /tidb/cdc/default/default/upstream/7365852681056034993 {"id":7365852681056034993,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2676.out cli tso query --pd=http://127.0.0.1:2379 < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:50:37 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ceaaeb5f-085d-49cd-8c49-2f7f0a197bff {"id":"ceaaeb5f-085d-49cd-8c49-2f7f0a197bff","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996234} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd8c9d30 ceaaeb5f-085d-49cd-8c49-2f7f0a197bff /tidb/cdc/default/default/upstream/7365852677201782411 {"id":7365852677201782411,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ceaaeb5f-085d-49cd-8c49-2f7f0a197bff {"id":"ceaaeb5f-085d-49cd-8c49-2f7f0a197bff","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996234} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd8c9d30 ceaaeb5f-085d-49cd-8c49-2f7f0a197bff /tidb/cdc/default/default/upstream/7365852677201782411 {"id":7365852677201782411,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ceaaeb5f-085d-49cd-8c49-2f7f0a197bff {"id":"ceaaeb5f-085d-49cd-8c49-2f7f0a197bff","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996234} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd8c9d30 ceaaeb5f-085d-49cd-8c49-2f7f0a197bff /tidb/cdc/default/default/upstream/7365852677201782411 {"id":7365852677201782411,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 71f341ea-c778-4a37-8b54-299f3679a87f Info: {"upstream_id":7365852677201782411,"namespace":"default","id":"71f341ea-c778-4a37-8b54-299f3679a87f","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:50:37.456806671+08:00","start_ts":449575972790075394,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575972790075394,"checkpoint_ts":449575972790075394,"checkpoint_time":"2024-05-06 19:50:34.093"} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f724140012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc, pid:1241, start at 2024-05-06 19:50:34.52918581 +0800 CST m=+5.288593686 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:34.537 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:34.501 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:34.501 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f727280005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc, pid:1317, start at 2024-05-06 19:50:34.703713687 +0800 CST m=+5.378330977 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:34.709 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:34.698 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:34.698 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } 0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:50:37 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/56179cce-039b-43c5-96f5-1410c1eccbe2 {"id":"56179cce-039b-43c5-96f5-1410c1eccbe2","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996234} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd8b0c33 56179cce-039b-43c5-96f5-1410c1eccbe2 /tidb/cdc/default/default/upstream/7365852678930291547 {"id":7365852678930291547,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/56179cce-039b-43c5-96f5-1410c1eccbe2 {"id":"56179cce-039b-43c5-96f5-1410c1eccbe2","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996234} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd8b0c33 56179cce-039b-43c5-96f5-1410c1eccbe2 /tidb/cdc/default/default/upstream/7365852678930291547 {"id":7365852678930291547,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/56179cce-039b-43c5-96f5-1410c1eccbe2 {"id":"56179cce-039b-43c5-96f5-1410c1eccbe2","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996234} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd8b0c33 56179cce-039b-43c5-96f5-1410c1eccbe2 /tidb/cdc/default/default/upstream/7365852678930291547 {"id":7365852678930291547,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 449575961219825665 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later pass test: create changefeed v2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:50:38 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d8ace4e-4f8e-411b-9f09-2b52f750fdad {"id":"0d8ace4e-4f8e-411b-9f09-2b52f750fdad","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996236} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd8d3431 0d8ace4e-4f8e-411b-9f09-2b52f750fdad /tidb/cdc/default/default/upstream/7365852689201857028 {"id":7365852689201857028,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d8ace4e-4f8e-411b-9f09-2b52f750fdad {"id":"0d8ace4e-4f8e-411b-9f09-2b52f750fdad","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996236} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd8d3431 0d8ace4e-4f8e-411b-9f09-2b52f750fdad /tidb/cdc/default/default/upstream/7365852689201857028 {"id":7365852689201857028,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d8ace4e-4f8e-411b-9f09-2b52f750fdad {"id":"0d8ace4e-4f8e-411b-9f09-2b52f750fdad","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996236} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd8d3431 0d8ace4e-4f8e-411b-9f09-2b52f750fdad /tidb/cdc/default/default/upstream/7365852689201857028 {"id":7365852689201857028,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + set +x + tso='449575973695782913 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449575973695782913 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:50:39 CST 2024] <<<<<< START cdc server in default_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.27192721.out server --log-file /tmp/tidb_cdc_test/default_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/default_value/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2694.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 1-th time... ***************** properties ***************** "dotransactions"="false" "requestdistribution"="uniform" "mysql.db"="consistent_replicate_ddl" "workload"="core" "updateproportion"="0" "scanproportion"="0" "insertproportion"="0" "fieldcount"="3" "threadcount"="10" "mysql.user"="root" "mysql.host"="127.0.0.1" "mysql.port"="4000" "readallfields"="true" "operationcount"="0" "recordcount"="50" "readproportion"="0" ********************************************** Run finished, takes 25.68476ms INSERT - Takes(s): 0.0, Count: 50, OPS: 8831.4, Avg(us): 4974, Min(us): 932, Max(us): 20147, 95th(us): 21000, 99th(us): 21000 pass test: remove changefeed v2 start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... check diff failed 2-th time, retry later [Mon May 6 19:50:39 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/ProcessorAddTableError=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.28002802.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:50:40 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.63066308.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449575974325714946 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449575974325714946 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:50:41 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.27342736.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 status code 200 pass test: show metadata status code 200 status code 200 pass test: delete service_gc_safepoint status code 500 pass test: resolve lock + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:50:42 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d67d8097-4b3e-40ca-bb46-0f1f67fc075c {"id":"d67d8097-4b3e-40ca-bb46-0f1f67fc075c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996239} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd9d6f36 d67d8097-4b3e-40ca-bb46-0f1f67fc075c /tidb/cdc/default/default/upstream/7365852700229380577 {"id":7365852700229380577,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d67d8097-4b3e-40ca-bb46-0f1f67fc075c {"id":"d67d8097-4b3e-40ca-bb46-0f1f67fc075c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996239} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd9d6f36 d67d8097-4b3e-40ca-bb46-0f1f67fc075c /tidb/cdc/default/default/upstream/7365852700229380577 {"id":7365852700229380577,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d67d8097-4b3e-40ca-bb46-0f1f67fc075c {"id":"d67d8097-4b3e-40ca-bb46-0f1f67fc075c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996239} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd9d6f36 d67d8097-4b3e-40ca-bb46-0f1f67fc075c /tidb/cdc/default/default/upstream/7365852700229380577 {"id":7365852700229380577,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2776.out cli changefeed create --start-ts=449575973695782913 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ table consistent_replicate_ddl.usertable1 not exists for 1-th check, retry later check diff successfully check_safepoint_forward http://127.0.0.1:2379 7365852681056034993 449575974589169668 449575973671403522 Create changefeed successfully! ID: 1b621683-e89f-4162-8f4b-5cd04ed46257 Info: {"upstream_id":7365852700229380577,"namespace":"default","id":"1b621683-e89f-4162-8f4b-5cd04ed46257","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:50:42.563942299+08:00","start_ts":449575973695782913,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575973695782913,"checkpoint_ts":449575973695782913,"checkpoint_time":"2024-05-06 19:50:37.548"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:50:42 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fb076ad2-d09e-4a88-a24f-e9d2e57b2bc5 {"id":"fb076ad2-d09e-4a88-a24f-e9d2e57b2bc5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996239} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd8978c6 fb076ad2-d09e-4a88-a24f-e9d2e57b2bc5 /tidb/cdc/default/default/upstream/7365852673421947846 {"id":7365852673421947846,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fb076ad2-d09e-4a88-a24f-e9d2e57b2bc5 {"id":"fb076ad2-d09e-4a88-a24f-e9d2e57b2bc5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996239} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd8978c6 fb076ad2-d09e-4a88-a24f-e9d2e57b2bc5 /tidb/cdc/default/default/upstream/7365852673421947846 {"id":7365852673421947846,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fb076ad2-d09e-4a88-a24f-e9d2e57b2bc5 {"id":"fb076ad2-d09e-4a88-a24f-e9d2e57b2bc5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996239} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd8978c6 fb076ad2-d09e-4a88-a24f-e9d2e57b2bc5 /tidb/cdc/default/default/upstream/7365852673421947846 {"id":7365852673421947846,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 dcd7a70a-d1ff-467d-93e6-53a17582e438 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=dcd7a70a-d1ff-467d-93e6-53a17582e438 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c dcd7a70a-d1ff-467d-93e6-53a17582e438 -s wait process cdc.test exit for 1-th time... + info='{ "upstream_id": 7365852673421947846, "namespace": "default", "id": "dcd7a70a-d1ff-467d-93e6-53a17582e438", "state": "normal", "checkpoint_tso": 449575975030095875, "checkpoint_time": "2024-05-06 19:50:42.638", "error": null }' + echo '{ "upstream_id": 7365852673421947846, "namespace": "default", "id": "dcd7a70a-d1ff-467d-93e6-53a17582e438", "state": "normal", "checkpoint_tso": 449575975030095875, "checkpoint_time": "2024-05-06 19:50:42.638", "error": null }' { "upstream_id": 7365852673421947846, "namespace": "default", "id": "dcd7a70a-d1ff-467d-93e6-53a17582e438", "state": "normal", "checkpoint_tso": 449575975030095875, "checkpoint_time": "2024-05-06 19:50:42.638", "error": null } ++ echo '{' '"upstream_id":' 7365852673421947846, '"namespace":' '"default",' '"id":' '"dcd7a70a-d1ff-467d-93e6-53a17582e438",' '"state":' '"normal",' '"checkpoint_tso":' 449575975030095875, '"checkpoint_time":' '"2024-05-06' '19:50:42.638",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365852673421947846, '"namespace":' '"default",' '"id":' '"dcd7a70a-d1ff-467d-93e6-53a17582e438",' '"state":' '"normal",' '"checkpoint_tso":' 449575975030095875, '"checkpoint_time":' '"2024-05-06' '19:50:42.638",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check diff failed 1-th time, retry later run task successfully check_changefeed_state http://127.0.0.1:2379 991e3956-33b6-407f-b612-58d61329a031 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=991e3956-33b6-407f-b612-58d61329a031 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 991e3956-33b6-407f-b612-58d61329a031 -s wait process cdc.test exit for 2-th time... Verifying downstream PD is started... + info='{ "upstream_id": 7365852681056034993, "namespace": "default", "id": "991e3956-33b6-407f-b612-58d61329a031", "state": "stopped", "checkpoint_tso": 449575975113457669, "checkpoint_time": "2024-05-06 19:50:42.956", "error": null }' + echo '{ "upstream_id": 7365852681056034993, "namespace": "default", "id": "991e3956-33b6-407f-b612-58d61329a031", "state": "stopped", "checkpoint_tso": 449575975113457669, "checkpoint_time": "2024-05-06 19:50:42.956", "error": null }' { "upstream_id": 7365852681056034993, "namespace": "default", "id": "991e3956-33b6-407f-b612-58d61329a031", "state": "stopped", "checkpoint_tso": 449575975113457669, "checkpoint_time": "2024-05-06 19:50:42.956", "error": null } ++ echo '{' '"upstream_id":' 7365852681056034993, '"namespace":' '"default",' '"id":' '"991e3956-33b6-407f-b612-58d61329a031",' '"state":' '"stopped",' '"checkpoint_tso":' 449575975113457669, '"checkpoint_time":' '"2024-05-06' '19:50:42.956",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365852681056034993, '"namespace":' '"default",' '"id":' '"991e3956-33b6-407f-b612-58d61329a031",' '"state":' '"stopped",' '"checkpoint_tso":' 449575975113457669, '"checkpoint_time":' '"2024-05-06' '19:50:42.956",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365852681056034993 wait process cdc.test exit for 3-th time... + set +x go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading golang.org/x/time v0.5.0 go: downloading golang.org/x/sync v0.7.0 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/coreos/go-semver v0.3.1 table consistent_replicate_ddl.usertable1 exists table consistent_replicate_ddl.usertable2 not exists for 1-th check, retry later < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:50:44 GMT < Content-Length: 895 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a32a784a-ac25-4e03-9513-73801be14d75 {"id":"a32a784a-ac25-4e03-9513-73801be14d75","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996241} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbcd99169 a32a784a-ac25-4e03-9513-73801be14d75 /tidb/cdc/default/default/upstream/7365852492112483417 {"id":7365852492112483417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a32a784a-ac25-4e03-9513-73801be14d75 {"id":"a32a784a-ac25-4e03-9513-73801be14d75","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996241} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbcd99169 a32a784a-ac25-4e03-9513-73801be14d75 /tidb/cdc/default/default/upstream/7365852492112483417 {"id":7365852492112483417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a32a784a-ac25-4e03-9513-73801be14d75 {"id":"a32a784a-ac25-4e03-9513-73801be14d75","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996241} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbcd99169 a32a784a-ac25-4e03-9513-73801be14d75 /tidb/cdc/default/default/upstream/7365852492112483417 {"id":7365852492112483417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6359.out cli changefeed create --start-ts=0 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-initialize-error cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:50:44 CST 2024] <<<<<< run test case http_api_tls success! >>>>>> go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/sys v0.19.0 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/text v0.14.0 Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7365852492112483417,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:50:44.87186897+08:00","start_ts":449575975585316865,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575975585316865,"checkpoint_ts":449575975585316865,"checkpoint_time":"2024-05-06 19:50:44.756"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:50:45 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0435d3fd-fd15-46d9-a1e2-d896daaedb16 {"id":"0435d3fd-fd15-46d9-a1e2-d896daaedb16","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996241} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd9ca93b 0435d3fd-fd15-46d9-a1e2-d896daaedb16 /tidb/cdc/default/default/upstream/7365852703802375758 {"id":7365852703802375758,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0435d3fd-fd15-46d9-a1e2-d896daaedb16 {"id":"0435d3fd-fd15-46d9-a1e2-d896daaedb16","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996241} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd9ca93b 0435d3fd-fd15-46d9-a1e2-d896daaedb16 /tidb/cdc/default/default/upstream/7365852703802375758 {"id":7365852703802375758,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0435d3fd-fd15-46d9-a1e2-d896daaedb16 {"id":"0435d3fd-fd15-46d9-a1e2-d896daaedb16","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996241} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd9ca93b 0435d3fd-fd15-46d9-a1e2-d896daaedb16 /tidb/cdc/default/default/upstream/7365852703802375758 {"id":7365852703802375758,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2795.out cli changefeed create --start-ts=449575974325714946 --sink-uri=mysql://normal:123456@127.0.0.1:3306/tidb-txn-mode=pessimistic check diff failed 2-th time, retry later Starting Upstream TiDB... Create changefeed successfully! ID: 98799d1b-4577-403a-ad92-79c4b7188771 Info: {"upstream_id":7365852703802375758,"namespace":"default","id":"98799d1b-4577-403a-ad92-79c4b7188771","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/tidb-txn-mode=pessimistic","create_time":"2024-05-06T19:50:45.991548142+08:00","start_ts":449575974325714946,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575974325714946,"checkpoint_ts":449575974325714946,"checkpoint_time":"2024-05-06 19:50:39.951"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable2 not exists for 2-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449575975585316865, "checkpoint_time": "2024-05-06 19:50:44.756", "error": { "time": "2024-05-06T19:50:45.050499432+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449575975585316865, "checkpoint_time": "2024-05-06 19:50:44.756", "error": { "time": "2024-05-06T19:50:45.050499432+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449575975585316865, "checkpoint_time": "2024-05-06 19:50:44.756", "error": { "time": "2024-05-06T19:50:45.050499432+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365852492112483417, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449575975585316865, '"checkpoint_time":' '"2024-05-06' '19:50:44.756",' '"error":' '{' '"time":' '"2024-05-06T19:50:45.050499432+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading cloud.google.com/go v0.112.2 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/tools v0.20.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/dolthub/swiss v0.2.1 run task successfully go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 check_changefeed_state http://127.0.0.1:2379 991e3956-33b6-407f-b612-58d61329a031 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=991e3956-33b6-407f-b612-58d61329a031 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 991e3956-33b6-407f-b612-58d61329a031 -s go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 + set +x go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 + info='{ "upstream_id": 7365852681056034993, "namespace": "default", "id": "991e3956-33b6-407f-b612-58d61329a031", "state": "normal", "checkpoint_tso": 449575975113457669, "checkpoint_time": "2024-05-06 19:50:42.956", "error": null }' + echo '{ "upstream_id": 7365852681056034993, "namespace": "default", "id": "991e3956-33b6-407f-b612-58d61329a031", "state": "normal", "checkpoint_tso": 449575975113457669, "checkpoint_time": "2024-05-06 19:50:42.956", "error": null }' { "upstream_id": 7365852681056034993, "namespace": "default", "id": "991e3956-33b6-407f-b612-58d61329a031", "state": "normal", "checkpoint_tso": 449575975113457669, "checkpoint_time": "2024-05-06 19:50:42.956", "error": null } ++ echo '{' '"upstream_id":' 7365852681056034993, '"namespace":' '"default",' '"id":' '"991e3956-33b6-407f-b612-58d61329a031",' '"state":' '"normal",' '"checkpoint_tso":' 449575975113457669, '"checkpoint_time":' '"2024-05-06' '19:50:42.956",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365852681056034993, '"namespace":' '"default",' '"id":' '"991e3956-33b6-407f-b612-58d61329a031",' '"state":' '"normal",' '"checkpoint_tso":' 449575975113457669, '"checkpoint_time":' '"2024-05-06' '19:50:42.956",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7365852681056034993 449575975113457668 449575975113457669 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da check diff successfully go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/google/btree v1.1.2 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable2 exists table consistent_replicate_ddl.usertable3 not exists for 1-th check, retry later go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 wait process cdc.test exit for 1-th time... go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 run task successfully go: downloading golang.org/x/sys v0.19.0 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s check_changefeed_state http://127.0.0.1:2379 991e3956-33b6-407f-b612-58d61329a031 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=991e3956-33b6-407f-b612-58d61329a031 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 991e3956-33b6-407f-b612-58d61329a031 -s go: downloading golang.org/x/text v0.14.0 wait process cdc.test exit for 2-th time... + info='{ "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449575975585316865, "checkpoint_time": "2024-05-06 19:50:44.756", "error": { "time": "2024-05-06T19:50:45.050499432+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449575975585316865, "checkpoint_time": "2024-05-06 19:50:44.756", "error": { "time": "2024-05-06T19:50:45.050499432+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449575975585316865, "checkpoint_time": "2024-05-06 19:50:44.756", "error": { "time": "2024-05-06T19:50:45.050499432+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365852492112483417, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449575975585316865, '"checkpoint_time":' '"2024-05-06' '19:50:44.756",' '"error":' '{' '"time":' '"2024-05-06T19:50:45.050499432+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later + info='{ "upstream_id": 7365852681056034993, "namespace": "default", "id": "991e3956-33b6-407f-b612-58d61329a031", "state": "stopped", "checkpoint_tso": 449575975113457669, "checkpoint_time": "2024-05-06 19:50:42.956", "error": null }' + echo '{ "upstream_id": 7365852681056034993, "namespace": "default", "id": "991e3956-33b6-407f-b612-58d61329a031", "state": "stopped", "checkpoint_tso": 449575975113457669, "checkpoint_time": "2024-05-06 19:50:42.956", "error": null }' { "upstream_id": 7365852681056034993, "namespace": "default", "id": "991e3956-33b6-407f-b612-58d61329a031", "state": "stopped", "checkpoint_tso": 449575975113457669, "checkpoint_time": "2024-05-06 19:50:42.956", "error": null } ++ echo '{' '"upstream_id":' 7365852681056034993, '"namespace":' '"default",' '"id":' '"991e3956-33b6-407f-b612-58d61329a031",' '"state":' '"stopped",' '"checkpoint_tso":' 449575975113457669, '"checkpoint_time":' '"2024-05-06' '19:50:42.956",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365852681056034993, '"namespace":' '"default",' '"id":' '"991e3956-33b6-407f-b612-58d61329a031",' '"state":' '"stopped",' '"checkpoint_tso":' 449575975113457669, '"checkpoint_time":' '"2024-05-06' '19:50:42.956",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 3-th time... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 17.44 secs (210274313 bytes/sec) [Pipeline] { check_changefeed_state http://127.0.0.1:2379 ed3fce16-b7ed-41c9-a93c-0a5dfbcd8c8b normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=ed3fce16-b7ed-41c9-a93c-0a5dfbcd8c8b + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c ed3fce16-b7ed-41c9-a93c-0a5dfbcd8c8b -s [Pipeline] cache + info='{ "upstream_id": 7365852681056034993, "namespace": "default", "id": "ed3fce16-b7ed-41c9-a93c-0a5dfbcd8c8b", "state": "normal", "checkpoint_tso": 449575976699166725, "checkpoint_time": "2024-05-06 19:50:49.005", "error": null }' + echo '{ "upstream_id": 7365852681056034993, "namespace": "default", "id": "ed3fce16-b7ed-41c9-a93c-0a5dfbcd8c8b", "state": "normal", "checkpoint_tso": 449575976699166725, "checkpoint_time": "2024-05-06 19:50:49.005", "error": null }' { "upstream_id": 7365852681056034993, "namespace": "default", "id": "ed3fce16-b7ed-41c9-a93c-0a5dfbcd8c8b", "state": "normal", "checkpoint_tso": 449575976699166725, "checkpoint_time": "2024-05-06 19:50:49.005", "error": null } ++ echo '{' '"upstream_id":' 7365852681056034993, '"namespace":' '"default",' '"id":' '"ed3fce16-b7ed-41c9-a93c-0a5dfbcd8c8b",' '"state":' '"normal",' '"checkpoint_tso":' 449575976699166725, '"checkpoint_time":' '"2024-05-06' '19:50:49.005",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365852681056034993, '"namespace":' '"default",' '"id":' '"ed3fce16-b7ed-41c9-a93c-0a5dfbcd8c8b",' '"state":' '"normal",' '"checkpoint_tso":' 449575976699166725, '"checkpoint_time":' '"2024-05-06' '19:50:49.005",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365852681056034993 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:50:49 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.19.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable3 not exists for 2-th check, retry later go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading k8s.io/apimachinery v0.28.6 table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable3 exists table consistent_replicate_ddl.usertable_bak not exists for 1-th check, retry later run task successfully table test.finish_mark not exists for 2-th check, retry later Changefeed remove successfully. ID: 991e3956-33b6-407f-b612-58d61329a031 CheckpointTs: 449575975113457669 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_forward http://127.0.0.1:2379 7365852681056034993 449575976699166724 449575975113457669 449575976699166725 check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449575977590718466, "checkpoint_time": "2024-05-06 19:50:52.406", "error": null }' + echo '{ "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449575977590718466, "checkpoint_time": "2024-05-06 19:50:52.406", "error": null }' { "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449575977590718466, "checkpoint_time": "2024-05-06 19:50:52.406", "error": null } ++ echo '{' '"upstream_id":' 7365852492112483417, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449575977590718466, '"checkpoint_time":' '"2024-05-06' '19:50:52.406",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365852492112483417, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449575977590718466, '"checkpoint_time":' '"2024-05-06' '19:50:52.406",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6543.out cli changefeed pause -c changefeed-initialize-error go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... run task successfully Changefeed remove successfully. ID: ed3fce16-b7ed-41c9-a93c-0a5dfbcd8c8b CheckpointTs: 449575977747742721 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_cleared http://127.0.0.1:2379 7365852681056034993 run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f82e10000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb, pid:9348, start at 2024-05-06 19:50:51.537108 +0800 CST m=+5.388832419 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:51.545 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:51.524 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:51.524 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f82e10000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb, pid:9348, start at 2024-05-06 19:50:51.537108 +0800 CST m=+5.388832419 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:51.545 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:51.524 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:51.524 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f82ec40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-s6bdb, pid:9423, start at 2024-05-06 19:50:51.598606448 +0800 CST m=+5.388140382 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:52:51.609 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:50:51.569 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:40:51.569 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release table consistent_replicate_ddl.usertable_bak not exists for 2-th check, retry later Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s wait process cdc.test exit for 1-th time... + info='{ "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449575977590718466, "checkpoint_time": "2024-05-06 19:50:52.406", "error": { "time": "2024-05-06T19:50:45.050499432+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449575977590718466, "checkpoint_time": "2024-05-06 19:50:52.406", "error": { "time": "2024-05-06T19:50:45.050499432+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449575977590718466, "checkpoint_time": "2024-05-06 19:50:52.406", "error": { "time": "2024-05-06T19:50:45.050499432+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365852492112483417, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449575977590718466, '"checkpoint_time":' '"2024-05-06' '19:50:52.406",' '"error":' '{' '"time":' '"2024-05-06T19:50:45.050499432+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365852492112483417, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449575977590718466, '"checkpoint_time":' '"2024-05-06' '19:50:52.406",' '"error":' '{' '"time":' '"2024-05-06T19:50:45.050499432+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6623.out cli changefeed resume -c changefeed-initialize-error table test.finish_mark not exists for 3-th check, retry later go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 wait process cdc.test exit for 2-th time... PASS coverage: 2.3% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 3-th time... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10674.out cli tso query --pd=http://127.0.0.1:2379 table consistent_replicate_ddl.usertable_bak exists cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:50:56 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/api_v2/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table test.finish_mark exists check table exists success check diff successfully + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449575977590718466, "checkpoint_time": "2024-05-06 19:50:52.406", "error": null }' + echo '{ "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449575977590718466, "checkpoint_time": "2024-05-06 19:50:52.406", "error": null }' { "upstream_id": 7365852492112483417, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449575977590718466, "checkpoint_time": "2024-05-06 19:50:52.406", "error": null } ++ echo '{' '"upstream_id":' 7365852492112483417, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449575977590718466, '"checkpoint_time":' '"2024-05-06' '19:50:52.406",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365852492112483417, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449575977590718466, '"checkpoint_time":' '"2024-05-06' '19:50:52.406",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6717.out cli changefeed remove -c changefeed-initialize-error wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... + set +x + tso='449575978748870657 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449575978748870657 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:50:58 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.1071910721.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 449575978901438467 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/api_v2 Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:50:58 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + set +x wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:51:01 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ade62f66-dab5-43ec-b0e7-1be133107a63 {"id":"ade62f66-dab5-43ec-b0e7-1be133107a63","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996258} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbdde933a ade62f66-dab5-43ec-b0e7-1be133107a63 /tidb/cdc/default/default/upstream/7365852772878808326 {"id":7365852772878808326,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ade62f66-dab5-43ec-b0e7-1be133107a63 {"id":"ade62f66-dab5-43ec-b0e7-1be133107a63","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996258} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbdde933a ade62f66-dab5-43ec-b0e7-1be133107a63 /tidb/cdc/default/default/upstream/7365852772878808326 {"id":7365852772878808326,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ade62f66-dab5-43ec-b0e7-1be133107a63 {"id":"ade62f66-dab5-43ec-b0e7-1be133107a63","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996258} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbdde933a ade62f66-dab5-43ec-b0e7-1be133107a63 /tidb/cdc/default/default/upstream/7365852772878808326 {"id":7365852772878808326,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10774.out cli changefeed create --start-ts=449575978748870657 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: d606be79-864f-49b5-8b78-a0d958b3d506 Info: {"upstream_id":7365852772878808326,"namespace":"default","id":"d606be79-864f-49b5-8b78-a0d958b3d506","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:51:01.864117081+08:00","start_ts":449575978748870657,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575978748870657,"checkpoint_ts":449575978748870657,"checkpoint_time":"2024-05-06 19:50:56.824"} PASS cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:51:01 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release coverage: 2.6% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:51:03 CST 2024] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true);github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.29202922.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server2 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + set +x + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:51:06 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/a871fb9b-a841-4eba-beff-4f5387622ebb {UpstreamID:7365852689201857028 Namespace:default ID:a871fb9b-a841-4eba-beff-4f5387622ebb SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:50:38.960242262 +0800 CST StartTs:449575974035783683 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035a4a20 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449575974061998083} {CheckpointTs:449575980982599684 MinTableBarrierTs:449575980982599684 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575980982599684, checkpointTs: 449575980982599684, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449575980982599684, checkpointTs: 449575980982599684, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449575980982599684, checkpointTs: 449575980982599684, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575980982599684, checkpointTs: 449575980982599684, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449575980982599684, checkpointTs: 449575980982599684, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/07a56886-3188-4c2e-bfa0-079706958756 {"id":"07a56886-3188-4c2e-bfa0-079706958756","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996263} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd8d3563 07a56886-3188-4c2e-bfa0-079706958756 /tidb/cdc/default/default/changefeed/info/a871fb9b-a841-4eba-beff-4f5387622ebb {"upstream-id":7365852689201857028,"namespace":"default","changefeed-id":"a871fb9b-a841-4eba-beff-4f5387622ebb","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:50:38.960242262+08:00","start-ts":449575974035783683,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575974061998083} /tidb/cdc/default/default/changefeed/status/a871fb9b-a841-4eba-beff-4f5387622ebb {"checkpoint-ts":449575980982599684,"min-table-barrier-ts":449575980982599684,"admin-job-type":0} /tidb/cdc/default/default/task/position/07a56886-3188-4c2e-bfa0-079706958756/a871fb9b-a841-4eba-beff-4f5387622ebb {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365852689201857028 {"id":7365852689201857028,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/a871fb9b-a841-4eba-beff-4f5387622ebb {UpstreamID:7365852689201857028 Namespace:default ID:a871fb9b-a841-4eba-beff-4f5387622ebb SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:50:38.960242262 +0800 CST StartTs:449575974035783683 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035a4a20 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449575974061998083} {CheckpointTs:449575980982599684 MinTableBarrierTs:449575980982599684 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575980982599684, checkpointTs: 449575980982599684, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449575980982599684, checkpointTs: 449575980982599684, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449575980982599684, checkpointTs: 449575980982599684, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575980982599684, checkpointTs: 449575980982599684, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449575980982599684, checkpointTs: 449575980982599684, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/07a56886-3188-4c2e-bfa0-079706958756 {"id":"07a56886-3188-4c2e-bfa0-079706958756","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996263} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd8d3563 07a56886-3188-4c2e-bfa0-079706958756 /tidb/cdc/default/default/changefeed/info/a871fb9b-a841-4eba-beff-4f5387622ebb {"upstream-id":7365852689201857028,"namespace":"default","changefeed-id":"a871fb9b-a841-4eba-beff-4f5387622ebb","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:50:38.960242262+08:00","start-ts":449575974035783683,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"ch+ grep -q 'failed to get info:' angefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575974061998083} /tidb/cdc/default/default/changefeed/status/a871fb9b-a841-4eba-beff-4f5387622ebb {"checkpoint-ts":449575980982599684,"min-table-barrier-ts":449575980982599684,"admin-job-type":0} /tidb/cdc/default/default/task/position/07a56886-3188-4c2e-bfa0-079706958756/a871fb9b-a841-4eba-beff-4f5387622ebb {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365852689201857028 {"id":7365852689201857028,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/a871fb9b-a841-4eba-beff-4f5387622ebb {UpstreamID:7365852689201857028 Namespace:default ID:a871fb9b-a841-4eba-beff-4f5387622ebb SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:50:38.960242262 +0800 CST StartTs:449575974035783683 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035a4a20 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449575974061998083} {CheckpointTs:449575980982599684 MinTableBarrierTs:449575980982599684 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575980982599684, checkpointTs: 449575980982599684, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449575980982599684, checkpointTs: 449575980982599684, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449575980982599684, checkpointTs: 449575980982599684, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575980982599684, checkpointTs: 449575980982599684, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449575980982599684, checkpointTs: 449575980982599684, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/07a56886-3188-4c2e-bfa0-079706958756 {"id":"07a56886-3188-4c2e-bfa0-079706958756","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996263} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbd8d3563 07a56886-3188-4c2e-bfa0-079706958756 /tidb/cdc/default/default/changefeed/info/a871fb9b-a841-4eba-beff-4f5387622ebb {"upstream-id":7365852689201857028,"namespace":"default","changefeed-id":"a871fb9b-a841-4eba-beff-4f5387622ebb","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:50:38.960242262+08:00","start-ts":449575974035783683,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"ch+ grep -q 'etcd info' angefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575974061998083} /tidb/cdc/default/default/changefeed/status/a871fb9b-a841-4eba-beff-4f5387622ebb {"checkpoint-ts":449575980982599684,"min-table-barrier-ts":449575980982599684,"admin-job-type":0} /tidb/cdc/default/default/task/position/07a56886-3188-4c2e-bfa0-079706958756/a871fb9b-a841-4eba-beff-4f5387622ebb {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365852689201857028 {"id":7365852689201857028,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 15.61 secs (234906594 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f941740012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:7463, start at 2024-05-06 19:51:09.183442171 +0800 CST m=+5.385470263 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:53:09.193 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:51:09.149 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:41:09.149 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f941740012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:7463, start at 2024-05-06 19:51:09.183442171 +0800 CST m=+5.385470263 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:53:09.193 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:51:09.149 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:41:09.149 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f942300013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:7541, start at 2024-05-06 19:51:09.230052136 +0800 CST m=+5.376482781 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:53:09.242 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:51:09.247 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:41:09.247 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/api_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/api_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/api_v2/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/api_v2/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/api_v2/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... [Mon May 6 19:51:14 CST 2024] <<<<<< START cdc server in api_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.api_v2.87948796.out server --log-file /tmp/tidb_cdc_test/api_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/api_v2/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f974680004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:4108, start at 2024-05-06 19:51:12.415387366 +0800 CST m=+5.302680961 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:53:12.422 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:51:12.410 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:41:12.410 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f974680004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:4108, start at 2024-05-06 19:51:12.415387366 +0800 CST m=+5.302680961 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:53:12.422 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:51:12.410 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:41:12.410 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f975dc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:4193, start at 2024-05-06 19:51:12.526833005 +0800 CST m=+5.345895718 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:53:12.533 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:51:12.503 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:41:12.503 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:51:16 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 19:51:17 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.54875489.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:51:17 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2a0dd56d-b6fb-463b-bf82-b9064211d06f {"id":"2a0dd56d-b6fb-463b-bf82-b9064211d06f","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996275} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe27853f 2a0dd56d-b6fb-463b-bf82-b9064211d06f /tidb/cdc/default/default/upstream/7365852850123780035 {"id":7365852850123780035,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2a0dd56d-b6fb-463b-bf82-b9064211d06f {"id":"2a0dd56d-b6fb-463b-bf82-b9064211d06f","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996275} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe27853f 2a0dd56d-b6fb-463b-bf82-b9064211d06f /tidb/cdc/default/default/upstream/7365852850123780035 {"id":7365852850123780035,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2a0dd56d-b6fb-463b-bf82-b9064211d06f {"id":"2a0dd56d-b6fb-463b-bf82-b9064211d06f","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996275} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe27853f 2a0dd56d-b6fb-463b-bf82-b9064211d06f /tidb/cdc/default/default/upstream/7365852850123780035 {"id":7365852850123780035,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: cf-blackhole Info: {"upstream_id":7365852850123780035,"namespace":"default","id":"cf-blackhole","sink_uri":"blackhole:","create_time":"2024-05-06T19:51:17.989210639+08:00","start_ts":449575984273817602,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575984273817602,"checkpoint_ts":449575984273817602,"checkpoint_time":"2024-05-06 19:51:17.900"} + endpoints=http://127.0.0.1:2379 + changefeed_id=cf-blackhole + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c cf-blackhole -s + info='{ "upstream_id": 7365852850123780035, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 449575984273817602, "checkpoint_time": "2024-05-06 19:51:17.900", "error": null }' + echo '{ "upstream_id": 7365852850123780035, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 449575984273817602, "checkpoint_time": "2024-05-06 19:51:17.900", "error": null }' { "upstream_id": 7365852850123780035, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 449575984273817602, "checkpoint_time": "2024-05-06 19:51:17.900", "error": null } ++ echo '{' '"upstream_id":' 7365852850123780035, '"namespace":' '"default",' '"id":' '"cf-blackhole",' '"state":' '"normal",' '"checkpoint_tso":' 449575984273817602, '"checkpoint_time":' '"2024-05-06' '19:51:17.900",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365852850123780035, '"namespace":' '"default",' '"id":' '"cf-blackhole",' '"state":' '"normal",' '"checkpoint_tso":' 449575984273817602, '"checkpoint_time":' '"2024-05-06' '19:51:17.900",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + GO111MODULE=on + go run main.go model.go request.go cases.go go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/google/uuid v1.6.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading golang.org/x/sync v0.7.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/YangKeao/seahash v0.0.0-20240229041150-e7bf269c3140 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/philhofer/fwd v1.1.1 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.33.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/aws/smithy-go v1.13.5 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading golang.org/x/sys v0.19.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:51:20 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92aa8246-7958-4212-9d42-98b0d99e040c {"id":"92aa8246-7958-4212-9d42-98b0d99e040c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996277} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe366931 92aa8246-7958-4212-9d42-98b0d99e040c /tidb/cdc/default/default/upstream/7365852869764123223 {"id":7365852869764123223,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92aa8246-7958-4212-9d42-98b0d99e040c {"id":"92aa8246-7958-4212-9d42-98b0d99e040c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996277} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe366931 92aa8246-7958-4212-9d42-98b0d99e040c /tidb/cdc/default/default/upstream/7365852869764123223 {"id":7365852869764123223,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92aa8246-7958-4212-9d42-98b0d99e040c {"id":"92aa8246-7958-4212-9d42-98b0d99e040c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996277} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe366931 92aa8246-7958-4212-9d42-98b0d99e040c /tidb/cdc/default/default/upstream/7365852869764123223 {"id":7365852869764123223,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible table partition_table.t exists table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f9f128000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:4543, start at 2024-05-06 19:51:20.410093611 +0800 CST m=+5.245254294 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:53:20.418 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:51:20.394 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:41:20.394 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.t2 not exists for 2-th check, retry later ***************** properties ***************** "requestdistribution"="uniform" "dotransactions"="false" "operationcount"="0" "mysql.db"="changefeed_reconstruct" "mysql.host"="127.0.0.1" "recordcount"="50" "insertproportion"="0" "mysql.port"="4000" "mysql.user"="root" "scanproportion"="0" "readallfields"="true" "workload"="core" "readproportion"="0" "threadcount"="4" "updateproportion"="0" ********************************************** Run finished, takes 22.322928ms INSERT - Takes(s): 0.0, Count: 48, OPS: 2750.2, Avg(us): 1743, Min(us): 978, Max(us): 4493, 95th(us): 5000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/pingcap/fn v1.0.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/go-logr/stdr v1.2.2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36fa130c0022 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-mzzb9, pid:3840, start at 2024-05-06 19:51:22.61325286 +0800 CST m=+5.205077252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:53:22.621 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:51:22.613 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:41:22.613 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36fa130c0022 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-mzzb9, pid:3840, start at 2024-05-06 19:51:22.61325286 +0800 CST m=+5.205077252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:53:22.621 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:51:22.613 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:41:22.613 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36fa15540010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-mzzb9, pid:3920, start at 2024-05-06 19:51:22.730596957 +0800 CST m=+5.266914502 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:53:22.736 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:51:22.709 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:41:22.709 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f9f128000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:4543, start at 2024-05-06 19:51:20.410093611 +0800 CST m=+5.245254294 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:53:20.418 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:51:20.394 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:41:20.394 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36f9f2a00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:4624, start at 2024-05-06 19:51:20.516172547 +0800 CST m=+5.298071276 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:53:20.522 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:51:20.488 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:41:20.488 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later ***************** properties ***************** "scanproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "readproportion"="0" "requestdistribution"="uniform" "updateproportion"="0" "operationcount"="0" "insertproportion"="0" "workload"="core" "threadcount"="2" "recordcount"="10" "mysql.user"="root" "dotransactions"="false" "mysql.db"="testSync" "readallfields"="true" ********************************************** Run finished, takes 4.931044ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3195.0, Avg(us): 901, Min(us): 553, Max(us): 2095, 95th(us): 3000, 99th(us): 3000 table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later go: downloading k8s.io/apimachinery v0.28.6 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 14.67 secs (250079958 bytes/sec) [Pipeline] { [Pipeline] cache [Mon May 6 19:51:25 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.59405942.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 19:51:25 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.51695171.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/jmespath/go-jmespath v0.4.0 Verifying downstream PD is started... table partition_table.finish_mark not exists for 2-th check, retry later check diff successfully wait process 5492 exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: mysql... <<================= go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 wait process 5492 exit for 2-th time... wait process 5492 exit for 3-th time... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5492) - No such process wait process 5492 exit for 4-th time... process 5492 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 19:51:28 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.57945796.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.finish_mark not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:51:28 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a50dce22-a3e7-4d21-ac9d-6647d312d7b5 {"id":"a50dce22-a3e7-4d21-ac9d-6647d312d7b5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996285} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe51e337 a50dce22-a3e7-4d21-ac9d-6647d312d7b5 /tidb/cdc/default/default/upstream/7365852892577998637 {"id":7365852892577998637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a50dce22-a3e7-4d21-ac9d-6647d312d7b5 {"id":"a50dce22-a3e7-4d21-ac9d-6647d312d7b5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996285} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe51e337 a50dce22-a3e7-4d21-ac9d-6647d312d7b5 /tidb/cdc/default/default/upstream/7365852892577998637 {"id":7365852892577998637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a50dce22-a3e7-4d21-ac9d-6647d312d7b5 {"id":"a50dce22-a3e7-4d21-ac9d-6647d312d7b5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996285} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe51e337 a50dce22-a3e7-4d21-ac9d-6647d312d7b5 /tidb/cdc/default/default/upstream/7365852892577998637 {"id":7365852892577998637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:51:28 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97938ea6-1009-4c38-a183-e62dd361e10a {"id":"97938ea6-1009-4c38-a183-e62dd361e10a","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996286} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe609b29 97938ea6-1009-4c38-a183-e62dd361e10a /tidb/cdc/default/default/upstream/7365852913562389982 {"id":7365852913562389982,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97938ea6-1009-4c38-a183-e62dd361e10a {"id":"97938ea6-1009-4c38-a183-e62dd361e10a","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996286} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe609b29 97938ea6-1009-4c38-a183-e62dd361e10a /tidb/cdc/default/default/upstream/7365852913562389982 {"id":7365852913562389982,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97938ea6-1009-4c38-a183-e62dd361e10a {"id":"97938ea6-1009-4c38-a183-e62dd361e10a","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996286} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe609b29 97938ea6-1009-4c38-a183-e62dd361e10a /tidb/cdc/default/default/upstream/7365852913562389982 {"id":7365852913562389982,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.5224.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-manager Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7365852913562389982,"namespace":"default","id":"ddl-manager","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-06T19:51:29.228974871+08:00","start_ts":449575987213238278,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575987213238278,"checkpoint_ts":449575987213238278,"checkpoint_time":"2024-05-06 19:51:29.113"} PASS The 1 times to try to start tidb cluster... table changefeed_pause_resume.t1 not exists for 1-th check, retry later coverage: 2.6% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 4-th check, retry later + set +x table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:51:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/4f3d0df2-bc2e-4bf4-9509-34228ea4ea81 {UpstreamID:7365852869764123223 Namespace:default ID:4f3d0df2-bc2e-4bf4-9509-34228ea4ea81 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:51:20.724495867 +0800 CST StartTs:449575984984489987 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001726ea0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449575985010704386} {CheckpointTs:449575986190352386 MinTableBarrierTs:449575987501072387 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575986190352386, checkpointTs: 449575986190352386, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c2357b1b-f55e-4ee8-bc10-36a1aace7045 {"id":"c2357b1b-f55e-4ee8-bc10-36a1aace7045","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe3669d6 c2357b1b-f55e-4ee8-bc10-36a1aace7045 /tidb/cdc/default/default/changefeed/info/4f3d0df2-bc2e-4bf4-9509-34228ea4ea81 {"upstream-id":7365852869764123223,"namespace":"default","changefeed-id":"4f3d0df2-bc2e-4bf4-9509-34228ea4ea81","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:51:20.724495867+08:00","start-ts":449575984984489987,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575985010704386} /tidb/cdc/default/default/changefeed/status/4f3d0df2-bc2e-4bf4-9509-34228ea4ea81 {"checkpoint-ts":449575986190352386,"min-table-barrier-ts":449575987763216387,"admin-job-type":0} /tidb/cdc/default/default/task/position/c2357b1b-f55e-4ee8-bc10-36a1aace7045/4f3d0df2-bc2e-4bf4-9509-34228ea4ea81 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365852869764123223 {"id":7365852869764123223,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/4f3d0df2-bc2e-4bf4-9509-34228ea4ea81 {UpstreamID:7365852869764123223 Namespace:default ID:4f3d0df2-bc2e-4bf4-9509-34228ea4ea81 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:51:20.724495867 +0800 CST StartTs:449575984984489987 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001726ea0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449575985010704386} {CheckpointTs:449575986190352386 MinTableBarrierTs:449575987501072387 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575986190352386, checkpointTs: 449575986190352386, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c2357b1b-f55e-4ee8-bc10-36a1aace7045 {"id":"c2357b1b-f55e-4ee8-bc10-36a1aace7045","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe3669d6 c2357b1b-f55e-4ee8-bc10-36a1aace7045 /tidb/cdc/default/default/changefeed/info/4f3d0df2-bc2e-4bf4-9509-34228ea4ea81 {"upstream-id":7365852869764123223,"namespace":"default","changefeed-id":"4f3d0df2-bc2e-4bf4-9509-34228ea4ea81","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:51:20.724495867+08:00","start-ts":449575984984489987,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575985010704386} /tidb/cdc/default/default/changefeed/status/4f3d0df2-bc2e-4bf4-9509-34228ea4ea81 {"checkpoint-ts":449575986190352386,"min-table-barrier-ts":449575987763216387,"admin-job-type":0} /tidb/cdc/default/default/task/position/c2357b1b-f55e-4ee8-bc10-36a1aace7045/4f3d0df2-bc2e-4bf4-9509-34228ea4ea81 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365852869764123223 {"id":7365852869764123223,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/4f3d0df2-bc2e-4bf4-9509-34228ea4ea81 {UpstreamID:7365852869764123223 Namespace:default ID:4f3d0df2-bc2e-4bf4-9509-34228ea4ea81 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:51:20.724495867 +0800 CST StartTs:449575984984489987 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001726ea0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449575985010704386} {CheckpointTs:449575986190352386 MinTableBarrierTs:449575987501072387 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575986190352386, checkpointTs: 449575986190352386, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c2357b1b-f55e-4ee8-bc10-36a1aace7045 {"id":"c2357b1b-f55e-4ee8-bc10-36a1aace7045","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe3669d6 c2357b1b-f55e-4ee8-bc10-36a1aace7045 /tidb/cdc/default/default/changefeed/info/4f3d0df2-bc2e-4bf4-9509-34228ea4ea81 {"upstream-id":7365852869764123223,"namespace":"default","changefeed-id":"4f3d0df2-bc2e-4bf4-9509-34228ea4ea81","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:51:20.724495867+08:00","start-ts":449575984984489987,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575985010704386} /tidb/cdc/default/default/changefeed/status/4f3d0df2-bc2e-4bf4-9509-34228ea4ea81 {"checkpoint-ts":449575986190352386,"min-table-barrier-ts":449575987763216387,"admin-job-type":0} /tidb/cdc/default/default/task/position/c2357b1b-f55e-4ee8-bc10-36a1aace7045/4f3d0df2-bc2e-4bf4-9509-34228ea4ea81 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365852869764123223 {"id":7365852869764123223,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":nul+ grep -q 'etcd info' l}' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 5-th check, retry later "id": "c2357b1b-f55e-4ee8-bc10-36a1aace7045", "cluster-id": "default" run task successfully start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later table partition_table.finish_mark not exists for 6-th check, retry later capture_id: c2357b1b-f55e-4ee8-bc10-36a1aace7045 check_processor_table_count http://127.0.0.1:2379 4f3d0df2-bc2e-4bf4-9509-34228ea4ea81 c2357b1b-f55e-4ee8-bc10-36a1aace7045 1 run task successfully check_processor_table_count http://127.0.0.1:2379 4f3d0df2-bc2e-4bf4-9509-34228ea4ea81 c2357b1b-f55e-4ee8-bc10-36a1aace7045 0 table count 1 does equal to expected count 0 run task failed 1-th time, retry later table changefeed_pause_resume.t3 exists Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 7-th check, retry later check diff failed 1-th time, retry later check_processor_table_count http://127.0.0.1:2379 4f3d0df2-bc2e-4bf4-9509-34228ea4ea81 c2357b1b-f55e-4ee8-bc10-36a1aace7045 0 run task successfully Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 8-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36fad3fc001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl, pid:7741, start at 2024-05-06 19:51:34.95645898 +0800 CST m=+5.217739608 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:53:34.964 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:51:34.962 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:41:34.962 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36fad3fc001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl, pid:7741, start at 2024-05-06 19:51:34.95645898 +0800 CST m=+5.217739608 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:53:34.964 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:51:34.962 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:41:34.962 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36fae9e0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl, pid:7831, start at 2024-05-06 19:51:36.326226992 +0800 CST m=+6.512952703 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:53:36.336 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:51:36.312 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:41:36.312 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ***************** properties ***************** "scanproportion"="0" "workload"="core" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "mysql.db"="changefeed_reconstruct" "mysql.user"="root" "operationcount"="0" "updateproportion"="0" "threadcount"="4" "mysql.port"="4000" "readproportion"="0" "insertproportion"="0" "recordcount"="50" "readallfields"="true" "dotransactions"="false" ********************************************** Run finished, takes 32.235486ms INSERT - Takes(s): 0.0, Count: 45, OPS: 3366.4, Avg(us): 1495, Min(us): 912, Max(us): 18669, 95th(us): 2000, 99th(us): 19000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later check diff failed 2-th time, retry later table partition_table.finish_mark not exists for 9-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9063.out cli tso query --pd=http://127.0.0.1:2379 table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later check diff failed 3-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 13.55 secs (270677464 bytes/sec) [Pipeline] { [Pipeline] cache table partition_table.finish_mark not exists for 10-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449575990266953729 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449575990266953729 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:51:42 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.91139115.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully check diff failed 4-th time, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table partition_table.finish_mark exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully wait process cdc.test exit for 3-th time... wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:51:45 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:51:45 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1bbbe54-3752-4352-a053-368c6e02a224 {"id":"c1bbbe54-3752-4352-a053-368c6e02a224","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996302} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe88d53d c1bbbe54-3752-4352-a053-368c6e02a224 /tidb/cdc/default/default/upstream/7365852953413709767 {"id":7365852953413709767,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1bbbe54-3752-4352-a053-368c6e02a224 {"id":"c1bbbe54-3752-4352-a053-368c6e02a224","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996302} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe88d53d c1bbbe54-3752-4352-a053-368c6e02a224 /tidb/cdc/default/default/upstream/7365852953413709767 {"id":7365852953413709767,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1bbbe54-3752-4352-a053-368c6e02a224 {"id":"c1bbbe54-3752-4352-a053-368c6e02a224","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996302} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe88d53d c1bbbe54-3752-4352-a053-368c6e02a224 /tidb/cdc/default/default/upstream/7365852953413709767 {"id":7365852953413709767,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9173.out cli changefeed create --start-ts=449575990266953729 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: c35fa5d0-66fa-4dcb-87c2-da88d51586bd Info: {"upstream_id":7365852953413709767,"namespace":"default","id":"c35fa5d0-66fa-4dcb-87c2-da88d51586bd","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:51:45.755769774+08:00","start_ts":449575990266953729,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575990266953729,"checkpoint_ts":449575990266953729,"checkpoint_time":"2024-05-06 19:51:40.762"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... check diff failed 5-th time, retry later wait process cdc.test exit for 3-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36fb59d80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3n4n5, pid:4059, start at 2024-05-06 19:51:43.526265273 +0800 CST m=+5.658739569 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:53:43.536 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:51:43.528 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:41:43.528 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36fb59d80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3n4n5, pid:4059, start at 2024-05-06 19:51:43.526265273 +0800 CST m=+5.658739569 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:53:43.536 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:51:43.528 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:41:43.528 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36fb5824000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3n4n5, pid:4135, start at 2024-05-06 19:51:43.381061778 +0800 CST m=+5.462517378 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:53:43.393 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:51:43.369 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:41:43.369 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:51:46 CST 2024] <<<<<< run test case partition_table success! >>>>>> + set +x check diff successfully [Mon May 6 19:51:48 CST 2024] <<<<<< START cdc server in cdc case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.54925494.out server --log-file /tmp/tidb_cdc_test/cdc/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process 5174 exit for 1-th time... wait process 5174 exit for 2-th time... wait process 5174 exit for 3-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5174) - No such process wait process 5174 exit for 4-th time... process 5174 already exit [Mon May 6 19:51:47 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.53005302.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_sequence.finish_mark not exists for 1-th check, retry later check diff failed 1-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 5.22 secs (702471891 bytes/sec) [Pipeline] { [Pipeline] cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:51:51 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f59e79ed-c4d9-432e-a38f-1eda3f58fcd0 {"id":"f59e79ed-c4d9-432e-a38f-1eda3f58fcd0","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996308} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbeaca62e f59e79ed-c4d9-432e-a38f-1eda3f58fcd0 /tidb/cdc/default/default/upstream/7365852996344152437 {"id":7365852996344152437,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f59e79ed-c4d9-432e-a38f-1eda3f58fcd0 {"id":"f59e79ed-c4d9-432e-a38f-1eda3f58fcd0","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996308} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbeaca62e f59e79ed-c4d9-432e-a38f-1eda3f58fcd0 /tidb/cdc/default/default/upstream/7365852996344152437 {"id":7365852996344152437,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f59e79ed-c4d9-432e-a38f-1eda3f58fcd0 {"id":"f59e79ed-c4d9-432e-a38f-1eda3f58fcd0","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996308} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbeaca62e f59e79ed-c4d9-432e-a38f-1eda3f58fcd0 /tidb/cdc/default/default/upstream/7365852996344152437 {"id":7365852996344152437,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.5544.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config /tmp/tidb_cdc_test/cdc/pulsar_test.toml table ddl_sequence.finish_mark not exists for 2-th check, retry later check diff failed 2-th time, retry later Create changefeed successfully! ID: bf979cec-f193-4428-b500-e6515d0a4afe Info: {"upstream_id":7365852996344152437,"namespace":"default","id":"bf979cec-f193-4428-b500-e6515d0a4afe","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:51:51.726539557+08:00","start_ts":449575993115410435,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575993115410435,"checkpoint_ts":449575993115410435,"checkpoint_time":"2024-05-06 19:51:51.628"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... table ddl_sequence.finish_mark not exists for 3-th check, retry later + set +x go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 check diff failed 3-th time, retry later go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/text v0.14.0 < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:51:54 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365852913562389982 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 19:51:29.228974871 +0800 CST StartTs:449575987213238278 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00233d4d0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449575987239714818} {CheckpointTs:449575987907919876 MinTableBarrierTs:449575987907919876 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c0307d21-6279-4757-9ecc-659f499930db {"id":"c0307d21-6279-4757-9ecc-659f499930db","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996307} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe609d8b c0307d21-6279-4757-9ecc-659f499930db /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365852913562389982,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T19:51:29.228974871+08:00","start-ts":449575987213238278,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575987239714818} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449575987907919876,"min-table-barrier-ts":449575987907919876,"admin-job-type":0} /tidb/cdc/default/default/task/position/c0307d21-6279-4757-9ecc-659f499930db/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365852913562389982 {"id":7365852913562389982,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365852913562389982 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 19:51:29.228974871 +0800 CST StartTs:449575987213238278 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00233d4d0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449575987239714818} {CheckpointTs:449575987907919876 MinTableBarrierTs:449575987907919876 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c0307d21-6279-4757-9ecc-659f499930db {"id":"c0307d21-6279-4757-9ecc-659f499930db","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996307} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe609d8b c0307d21-6279-4757-9ecc-659f499930db /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365852913562389982,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T19:51:29.228974871+08:00","start-ts":449575987213238278,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575987239714818} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449575987907919876,"min-table-barrier-ts":449575987907919876,"admin-job-type":0} /tidb/cdc/default/default/task/position/c0307d21-6279-4757-9ecc-659f499930db/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365852913562389982 {"id":7365852913562389982,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365852913562389982 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 19:51:29.228974871 +0800 CST StartTs:449575987213238278 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00233d4d0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449575987239714818} {CheckpointTs:449575987907919876 MinTableBarrierTs:449575987907919876 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c0307d21-6279-4757-9ecc-659f499930db {"id":"c0307d21-6279-4757-9ecc-659f499930db","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996307} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe609d8b c0307d21-6279-4757-9ecc-659f499930db /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365852913562389982,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T19:51:29.228974871+08:00","start-ts":449575987213238278,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575987239714818} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449575987907919876,"min-table-barrier-ts":449575987907919876,"admin-job-type":0} /tidb/cdc/default/default/task/position/c0307d21-6279-4757-9ecc-659f499930db/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365852913562389982 {"id":7365852913562389982,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:51:54 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.53665368.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_sequence.finish_mark not exists for 4-th check, retry later check diff failed 4-th time, retry later go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/gorilla/mux v1.8.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 \033[0;36m<<< Run all test success >>>\033[0m go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/jfcg/sixb v1.3.8 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/robfig/cron v1.2.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 table ddl_sequence.finish_mark not exists for 5-th check, retry later check diff failed 5-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:51:58 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365852913562389982 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 19:51:29.228974871 +0800 CST StartTs:449575987213238278 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00233d4d0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449575987239714818} {CheckpointTs:449575987907919876 MinTableBarrierTs:449575987907919876 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c0307d21-6279-4757-9ecc-659f499930db {"id":"c0307d21-6279-4757-9ecc-659f499930db","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996307} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe609d8b c0307d21-6279-4757-9ecc-659f499930db /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365852913562389982,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T19:51:29.228974871+08:00","start-ts":449575987213238278,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575987239714818} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449575988039254030,"min-table-barrier-ts":449575988039254030,"admin-job-type":0} /tidb/cdc/default/default/task/position/c0307d21-6279-4757-9ecc-659f499930db/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365852913562389982 {"id":7365852913562389982,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365852913562389982 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 19:51:29.228974871 +0800 CST StartTs:449575987213238278 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00233d4d0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449575987239714818} {CheckpointTs:449575987907919876 MinTableBarrierTs:449575987907919876 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c0307d21-6279-4757-9ecc-659f499930db {"id":"c0307d21-6279-4757-9ecc-659f499930db","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996307} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe609d8b c0307d21-6279-4757-9ecc-659f499930db /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365852913562389982,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T19:51:29.228974871+08:00","start-ts":449575987213238278,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575987239714818} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449575988039254030,"min-table-barrier-ts":449575988039254030,"admin-job-type":0} /tidb/cdc/default/default/task/position/c0307d21-6279-4757-9ecc-659f499930db/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365852913562389982 + grep -q 'failed to get info:' {"id":7365852913562389982,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365852913562389982 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 19:51:29.228974871 +0800 CST StartTs:449575987213238278 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00233d4d0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449575987239714818} {CheckpointTs:449575987907919876 MinTableBarrierTs:449575987907919876 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c0307d21-6279-4757-9ecc-659f499930db {"id":"c0307d21-6279-4757-9ecc-659f499930db","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996307} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbe609d8b c0307d21-6279-4757-9ecc-659f499930db /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365852913562389982,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T19:51:29.228974871+08:00","start-ts":449575987213238278,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449575987239714818} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449575988039254030,"min-table-barrier-ts":449575988039254030,"admin-job-type":0} /tidb/cdc/default/default/task/position/c0307d21-6279-4757-9ecc-659f499930db/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365852913562389982 {"id":7365852913562389982,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later table ddl_sequence.finish_mark not exists for 6-th check, retry later check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 table ddl_manager.finish_mark not exists for 2-th check, retry later table ddl_sequence.finish_mark not exists for 7-th check, retry later check diff failed 1-th time, retry later go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd Starting Upstream TiDB... table ddl_manager.finish_mark not exists for 3-th check, retry later Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_sequence.finish_mark exists check diff successfully check diff failed 2-th time, retry later wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 4-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 12.46 secs (294476480 bytes/sec) [Pipeline] { [Pipeline] cache check diff failed 3-th time, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:52:06 CST 2024] <<<<<< run test case ddl_sequence success! >>>>>> table ddl_manager.finish_mark not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 6-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36fcd74c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:7216, start at 2024-05-06 19:52:07.912906151 +0800 CST m=+5.248206120 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:54:07.919 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:52:07.891 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:42:07.891 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 7-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36fcd74c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:7216, start at 2024-05-06 19:52:07.912906151 +0800 CST m=+5.248206120 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:54:07.919 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:52:07.891 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:42:07.891 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36fcd8b0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:7302, start at 2024-05-06 19:52:07.995368583 +0800 CST m=+5.277786074 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:54:08.001 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:52:07.980 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:42:07.980 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 6-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8563.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 8-th check, retry later check diff successfully + set +x + tso='449575998794235905 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449575998794235905 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "mysql.db"="multi_capture_1" "updateproportion"="0" "mysql.port"="4000" "requestdistribution"="uniform" "readproportion"="0" "threadcount"="2" "operationcount"="0" "insertproportion"="0" "recordcount"="10" "workload"="core" "mysql.user"="root" "readallfields"="true" "dotransactions"="false" "mysql.host"="127.0.0.1" "scanproportion"="0" ********************************************** Run finished, takes 20.172649ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1850.2, Avg(us): 3952, Min(us): 1082, Max(us): 14744, 95th(us): 15000, 99th(us): 15000 table ddl_manager.finish_mark not exists for 9-th check, retry later ***************** properties ***************** "readproportion"="0" "operationcount"="0" "requestdistribution"="uniform" "mysql.user"="root" "mysql.db"="multi_capture_2" "recordcount"="10" "scanproportion"="0" "threadcount"="2" "mysql.port"="4000" "readallfields"="true" "workload"="core" "updateproportion"="0" "dotransactions"="false" "mysql.host"="127.0.0.1" "insertproportion"="0" ********************************************** Run finished, takes 9.31996ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1814.8, Avg(us): 1788, Min(us): 1074, Max(us): 3723, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "insertproportion"="0" "mysql.port"="4000" "mysql.user"="root" "operationcount"="0" "readallfields"="true" "readproportion"="0" "mysql.host"="127.0.0.1" "updateproportion"="0" "workload"="core" "scanproportion"="0" "recordcount"="10" "dotransactions"="false" "threadcount"="2" "mysql.db"="multi_capture_3" "requestdistribution"="uniform" ********************************************** Run finished, takes 9.011577ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1945.3, Avg(us): 1741, Min(us): 1156, Max(us): 3845, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "scanproportion"="0" "mysql.user"="root" "mysql.db"="multi_capture_4" "operationcount"="0" "workload"="core" "recordcount"="10" "requestdistribution"="uniform" "insertproportion"="0" "threadcount"="2" "readproportion"="0" "mysql.port"="4000" "mysql.host"="127.0.0.1" "readallfields"="true" "updateproportion"="0" "dotransactions"="false" ********************************************** Run finished, takes 9.906073ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1680.6, Avg(us): 1829, Min(us): 1088, Max(us): 3924, 95th(us): 4000, 99th(us): 4000 [Mon May 6 19:52:15 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.86918693.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 10-th check, retry later check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:52:18 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7acbe2aa-7d42-4cbe-b5e2-5dba6d040827 {"id":"7acbe2aa-7d42-4cbe-b5e2-5dba6d040827","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996335} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf0f538c 7acbe2aa-7d42-4cbe-b5e2-5dba6d040827 /tidb/cdc/default/default/upstream/7365853102292039079 {"id":7365853102292039079,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7acbe2aa-7d42-4cbe-b5e2-5dba6d040827 {"id":"7acbe2aa-7d42-4cbe-b5e2-5dba6d040827","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996335} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf0f538c 7acbe2aa-7d42-4cbe-b5e2-5dba6d040827 /tidb/cdc/default/default/upstream/7365853102292039079 {"id":7365853102292039079,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7acbe2aa-7d42-4cbe-b5e2-5dba6d040827 {"id":"7acbe2aa-7d42-4cbe-b5e2-5dba6d040827","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996335} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf0f538c 7acbe2aa-7d42-4cbe-b5e2-5dba6d040827 /tidb/cdc/default/default/upstream/7365853102292039079 {"id":7365853102292039079,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:52:18 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.87548756.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 11-th check, retry later check diff failed 3-th time, retry later table testSync.usertable exists table testSync.simple1 exists table testSync.simple2 exists table ddl_manager.finish_mark not exists for 12-th check, retry later check diff failed 4-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:52:21 GMT < Content-Length: 1295 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/71209f7b-bbcb-40f9-9bba-3b39dfdf0f5e {"id":"71209f7b-bbcb-40f9-9bba-3b39dfdf0f5e","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996339} /tidb/cdc/default/__cdc_meta__/capture/7acbe2aa-7d42-4cbe-b5e2-5dba6d040827 {"id":"7acbe2aa-7d42-4cbe-b5e2-5dba6d040827","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996335} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf0f538c 7acbe2aa-7d42-4cbe-b5e2-5dba6d040827 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf0f53bf 71209f7b-bbcb-40f9-9bba-3b39dfdf0f5e /tidb/cdc/default/default/upstream/7365853102292039079 {"id":7365853102292039079,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/71209f7b-bbcb-40f9-9bba-3b39dfdf0f5e {"id":"71209f7b-bbcb-40f9-9bba-3b39dfdf0f5e","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996339} /tidb/cdc/default/__cdc_meta__/capture/7acbe2aa-7d42-4cbe-b5e2-5dba6d040827 {"id":"7acbe2aa-7d42-4cbe-b5e2-5dba6d040827","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996335} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf0f538c 7acbe2aa-7d42-4cbe-b5e2-5dba6d040827 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf0f53bf 71209f7b-bbcb-40f9-9bba-3b39dfdf0f5e /tidb/cdc/default/default/upstream/7365853102292039079 {"id":7365853102292039079,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/71209f7b-bbcb-40f9-9bba-3b39dfdf0f5e {"id":"71209f7b-bbcb-40f9-9bba-3b39dfdf0f5e","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996339} /tidb/cdc/default/__cdc_meta__/capture/7acbe2aa-7d42-4cbe-b5e2-5dba6d040827 {"id":"7acbe2aa-7d42-4cbe-b5e2-5dba6d040827","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996335} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf0f538c 7acbe2aa-7d42-4cbe-b5e2-5dba6d040827 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf0f53bf 71209f7b-bbcb-40f9-9bba-3b39dfdf0f5e /tidb/cdc/default/default/upstream/7365853102292039079 {"id":7365853102292039079,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:52:21 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.88098811.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 13-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... check diff failed 5-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:52:24 GMT < Content-Length: 1786 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/42d11387-2e19-4b03-badd-7b439eed50ed {"id":"42d11387-2e19-4b03-badd-7b439eed50ed","address":"127.0.0.1:8303","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996342} /tidb/cdc/default/__cdc_meta__/capture/71209f7b-bbcb-40f9-9bba-3b39dfdf0f5e {"id":"71209f7b-bbcb-40f9-9bba-3b39dfdf0f5e","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996339} /tidb/cdc/default/__cdc_meta__/capture/7acbe2aa-7d42-4cbe-b5e2-5dba6d040827 {"id":"7acbe2aa-7d42-4cbe-b5e2-5dba6d040827","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996335} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf0f538c 7acbe2aa-7d42-4cbe-b5e2-5dba6d040827 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf0f53bf 71209f7b-bbcb-40f9-9bba-3b39dfdf0f5e /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf0f53df 42d11387-2e19-4b03-badd-7b439eed50ed /tidb/cdc/default/default/upstream/7365853102292039079 {"id":7365853102292039079,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/42d11387-2e19-4b03-badd-7b439eed50ed {"id":"42d11387-2e19-4b03-badd-7b439eed50ed","address":"127.0.0.1:8303","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996342} /tidb/cdc/default/__cdc_meta__/capture/71209f7b-bbcb-40f9-9bba-3b39dfdf0f5e {"id":"71209f7b-bbcb-40f9-9bba-3b39dfdf0f5e","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996339} /tidb/cdc/default/__cdc_meta__/capture/7acbe2aa-7d42-4cbe-b5e2-5dba6d040827 {"id":"7acbe2aa-7d42-4cbe-b5e2-5dba6d040827","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996335} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf0f538c 7acbe2aa-7d42-4cbe-b5e2-5dba6d040827 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf0f53bf 71209f7b-bbcb-40f9-9bba-3b39dfdf0f5e /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf0f53df 42d11387-2e19-4b03-badd-7b439eed50ed /tidb/cdc/default/default/upstream/7365853102292039079 {"id":7365853102292039079,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/42d11387-2e19-4b03-badd-7b439eed50ed {"id":"42d11387-2e19-4b03-badd-7b439eed50ed","address":"127.0.0.1:8303","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996342} /tidb/cdc/default/__cdc_meta__/capture/71209f7b-bbcb-40f9-9bba-3b39dfdf0f5e {"id":"71209f7b-bbcb-40f9-9bba-3b39dfdf0f5e","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996339} /tidb/cdc/default/__cdc_meta__/capture/7acbe2aa-7d42-4cbe-b5e2-5dba6d040827 {"id":"7acbe2aa-7d42-4cbe-b5e2-5dba6d040827","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996335} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf0f538c 7acbe2aa-7d42-4cbe-b5e2-5dba6d040827 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf0f53bf 71209f7b-bbcb-40f9-9bba-3b39dfdf0f5e /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf0f53df 42d11387-2e19-4b03-badd-7b439eed50ed /tidb/cdc/default/default/upstream/7365853102292039079 {"id":7365853102292039079,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8869.out cli changefeed create --start-ts=449575998794235905 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 Create changefeed successfully! ID: 99041367-128e-4614-9655-d697a54fde50 Info: {"upstream_id":7365853102292039079,"namespace":"default","id":"99041367-128e-4614-9655-d697a54fde50","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:52:25.221072491+08:00","start_ts":449575998794235905,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449575998794235905,"checkpoint_ts":449575998794235905,"checkpoint_time":"2024-05-06 19:52:13.291"} PASS table ddl_manager.finish_mark not exists for 14-th check, retry later coverage: 2.6% of statements in github.com/pingcap/tiflow/... TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later check data successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:52:23 CST 2024] <<<<<< run test case ddl_puller_lag success! >>>>>> check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x table multi_capture_1.usertable not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 15-th check, retry later check diff failed 1-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 19.81 secs (185176589 bytes/sec) [Pipeline] { [Pipeline] cache table multi_capture_1.usertable exists table multi_capture_2.usertable not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 16-th check, retry later check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 17-th check, retry later table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 18-th check, retry later table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_capture_4.usertable exists check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 19-th check, retry later check diff failed 5-th time, retry later check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36fe85d40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl, pid:10295, start at 2024-05-06 19:52:35.477906641 +0800 CST m=+6.885048165 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:54:35.484 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:52:35.445 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:42:35.445 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36fe85d40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl, pid:10295, start at 2024-05-06 19:52:35.477906641 +0800 CST m=+6.885048165 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:54:35.484 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:52:35.445 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:42:35.445 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36fe6d800014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-vp2kl, pid:10367, start at 2024-05-06 19:52:33.91380281 +0800 CST m=+5.268656772 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:54:33.920 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:52:33.888 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:42:33.888 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ***************** properties ***************** "mysql.port"="4000" "dotransactions"="false" "mysql.host"="127.0.0.1" "readallfields"="true" "mysql.user"="root" "readproportion"="0" "insertproportion"="0" "scanproportion"="0" "recordcount"="20" "requestdistribution"="uniform" "mysql.db"="multi_capture_1" "threadcount"="2" "operationcount"="0" "updateproportion"="0" "workload"="core" ********************************************** Run finished, takes 12.201855ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1859.2, Avg(us): 1140, Min(us): 505, Max(us): 2003, 95th(us): 2000, 99th(us): 3000 ***************** properties ***************** "scanproportion"="0" "mysql.user"="root" "threadcount"="2" "workload"="core" "requestdistribution"="uniform" "insertproportion"="0" "mysql.db"="multi_capture_2" "dotransactions"="false" "mysql.port"="4000" "readallfields"="true" "operationcount"="0" "mysql.host"="127.0.0.1" "readproportion"="0" "updateproportion"="0" "recordcount"="20" ********************************************** Run finished, takes 11.391064ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1943.5, Avg(us): 1014, Min(us): 497, Max(us): 1879, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "scanproportion"="0" "updateproportion"="0" "readallfields"="true" "readproportion"="0" "mysql.db"="multi_capture_3" "requestdistribution"="uniform" "mysql.user"="root" "recordcount"="20" "mysql.host"="127.0.0.1" "workload"="core" "threadcount"="2" "operationcount"="0" "mysql.port"="4000" "insertproportion"="0" "dotransactions"="false" ********************************************** Run finished, takes 12.121711ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1848.2, Avg(us): 1025, Min(us): 494, Max(us): 1799, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "scanproportion"="0" "requestdistribution"="uniform" "operationcount"="0" "mysql.user"="root" "readallfields"="true" "insertproportion"="0" "dotransactions"="false" "recordcount"="20" "mysql.port"="4000" "workload"="core" "readproportion"="0" "threadcount"="2" "updateproportion"="0" "mysql.db"="multi_capture_4" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 11.065485ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2249.3, Avg(us): 1059, Min(us): 485, Max(us): 2159, 95th(us): 3000, 99th(us): 3000 check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 20-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... check diff failed 6-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.11622.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 21-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... check diff successfully wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='449576005611028481 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576005611028481 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:52:40 CST 2024] <<<<<< START cdc server in resourcecontrol case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.1166211664.out server --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resourcecontrol/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 22-th check, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:52:41 CST 2024] <<<<<< run test case multi_capture success! >>>>>> check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 23-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:52:43 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7ce5e466-aa2c-4941-b996-400434eb578f {"id":"7ce5e466-aa2c-4941-b996-400434eb578f","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996361} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf70cd37 7ce5e466-aa2c-4941-b996-400434eb578f /tidb/cdc/default/default/upstream/7365853217631130153 {"id":7365853217631130153,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7ce5e466-aa2c-4941-b996-400434eb578f {"id":"7ce5e466-aa2c-4941-b996-400434eb578f","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996361} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf70cd37 7ce5e466-aa2c-4941-b996-400434eb578f /tidb/cdc/default/default/upstream/7365853217631130153 {"id":7365853217631130153,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7ce5e466-aa2c-4941-b996-400434eb578f {"id":"7ce5e466-aa2c-4941-b996-400434eb578f","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996361} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbf70cd37 7ce5e466-aa2c-4941-b996-400434eb578f /tidb/cdc/default/default/upstream/7365853217631130153 {"id":7365853217631130153,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.11727.out cli changefeed create --start-ts=449576005611028481 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ check diff failed 2-th time, retry later Create changefeed successfully! ID: 91365301-1afc-4048-85e3-69efe761196f Info: {"upstream_id":7365853217631130153,"namespace":"default","id":"91365301-1afc-4048-85e3-69efe761196f","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:52:44.33298255+08:00","start_ts":449576005611028481,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576005611028481,"checkpoint_ts":449576005611028481,"checkpoint_time":"2024-05-06 19:52:39.295"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:52:36.700 +08:00] [INFO] [main.go:99] ["running ddl test: 0 modifyColumnDefaultValueDDL1"] [2024/05/06 19:52:36.700 +08:00] [INFO] [main.go:99] ["running ddl test: 1 modifyColumnDefaultValueDDL2"] [2024/05/06 19:52:36.793 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs36fbf938_db3d_41de_a11f_1553687d6614"] [2024/05/06 19:52:36.893 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc568c972_d010_4465_a38f_67179db7e2d8"] [2024/05/06 19:52:36.944 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs0bf0f36e_44ba_45bc_a4b2_a7dd081eb71a"] [2024/05/06 19:52:37.032 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs63ba590e_2a3c_4c31_aad6_ee98a9fd4dde"] [2024/05/06 19:52:37.065 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs02f134f3_748a_45a1_964f_7425fd939b86"] [2024/05/06 19:52:37.260 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs48c66071_bd21_47c6_9de8_ef5454a39b3f"] [2024/05/06 19:52:37.262 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd930fa37_0a7b_48f0_b6a9_b65327242778"] [2024/05/06 19:52:37.265 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsed774bb4_a14c_4b6a_b588_c47571bbfcd3"] [2024/05/06 19:52:37.557 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:52:37.564 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:52:37.765 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:52:37.880 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:52:37.978 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/06 19:52:38.061 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:52:38.172 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:52:38.174 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/06 19:52:38.179 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:52:38.259 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:52:38.366 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:52:38.378 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:52:38.462 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:52:38.467 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:52:38.468 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:52:38.482 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:52:38.484 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:52:38.568 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:52:38.575 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:52:38.662 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/06 19:52:38.679 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:52:38.766 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:52:38.781 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/06 19:52:38.964 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:52:38.983 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/06 19:52:39.070 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:52:39.262 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:52:39.262 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/06 19:52:39.263 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:52:39.276 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:52:39.377 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:52:39.387 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:52:39.477 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:52:39.567 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:52:39.569 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:52:39.575 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:52:39.659 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:52:39.659 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:52:39.687 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:52:39.770 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:52:39.982 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:52:40.071 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/06 19:52:40.077 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/06 19:52:40.159 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:52:40.168 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:52:40.175 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/06 19:52:40.262 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/06 19:52:40.267 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/06 19:52:40.360 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:52:40.375 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/06 19:52:40.375 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:52:40.385 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:52:40.471 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:52:40.564 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:52:40.573 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:52:40.668 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:52:40.671 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:52:40.679 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:52:40.684 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:52:40.772 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:52:40.859 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:52:40.876 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:52:41.069 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:52:41.177 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:52:41.272 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/06 19:52:41.279 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:52:41.470 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/06 19:52:41.471 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:52:41.481 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:52:41.559 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:52:41.560 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:52:41.569 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/06 19:52:41.665 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:52:41.668 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:52:41.678 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/06 19:52:41.759 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:52:41.775 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:52:41.777 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:52:41.865 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:52:41.883 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:52:41.883 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:52:41.976 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:52:42.161 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:52:42.264 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:52:42.362 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:52:42.371 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/06 19:52:42.561 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/06 19:52:42.571 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:52:42.670 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:52:42.676 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:52:42.678 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:52:42.866 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:52:42.870 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:52:42.885 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:52:42.962 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:52:42.964 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:52:43.072 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:52:43.082 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:52:43.162 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:52:43.169 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/06 19:52:43.173 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/06 19:52:43.259 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:52:43.263 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/06 19:52:43.268 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/06 19:52:43.360 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:52:43.471 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:52:43.481 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:52:43.660 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/06 19:52:43.766 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:52:43.769 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:52:43.773 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/06 19:52:43.865 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:52:43.869 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:52:43.975 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:52:43.988 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:52:44.063 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:52:44.077 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:52:44.081 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:52:44.190 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:52:44.272 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:52:44.371 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:52:44.466 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:52:44.563 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:52:44.663 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:52:44.672 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/06 19:52:44.680 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/06 19:52:44.767 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:52:44.864 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/05/06 19:52:44.868 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:52:44.876 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:52:45.060 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/05/06 19:52:45.077 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:52:45.085 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:52:45.159 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:52:45.164 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:52:45.166 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:52:45.270 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:52:45.284 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:52:45.285 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] table ddl_manager.finish_mark not exists for 24-th check, retry later [2024/05/06 19:52:45.376 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:52:45.567 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:52:45.664 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:52:45.675 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:52:45.763 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:52:45.870 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:52:45.959 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/05/06 19:52:45.962 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:52:45.971 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:52:46.079 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] + set +x [2024/05/06 19:52:46.169 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/06 19:52:46.175 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:52:46.176 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/06 19:52:46.180 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:52:46.257 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/06 19:52:46.259 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:52:46.259 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:52:46.263 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/06 19:52:46.272 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:52:46.283 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:52:46.368 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:52:46.464 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:52:46.470 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] check diff failed 3-th time, retry later [2024/05/06 19:52:46.670 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:52:46.762 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:52:46.767 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:52:46.872 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:52:46.965 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:52:46.972 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 25-th check, retry later [2024/05/06 19:52:47.174 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:52:47.258 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:52:47.260 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/06 19:52:47.358 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:52:47.362 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:52:47.382 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:52:47.383 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:52:47.467 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:52:47.474 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:52:47.478 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:52:47.672 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:52:47.674 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:52:47.767 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/05/06 19:52:47.773 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/05/06 19:52:47.782 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:52:47.860 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:52:47.878 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 16.17 secs (226871199 bytes/sec) [Pipeline] { [Pipeline] cache [2024/05/06 19:52:47.978 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:52:48.075 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/05/06 19:52:48.077 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:52:48.279 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:52:48.367 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:52:48.369 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/05/06 19:52:48.381 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:52:48.382 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] check diff failed 4-th time, retry later [2024/05/06 19:52:48.481 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:52:48.484 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:52:48.570 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:52:48.571 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:52:48.582 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] table resourcecontrol.finish_mark not exists for 1-th check, retry later [2024/05/06 19:52:48.772 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:52:48.775 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:52:48.961 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:52:48.963 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:52:48.979 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:52:49.168 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] table ddl_manager.finish_mark not exists for 26-th check, retry later [2024/05/06 19:52:49.179 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/05/06 19:52:49.271 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:52:49.369 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/06 19:52:49.374 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/06 19:52:49.374 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/05/06 19:52:49.380 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/05/06 19:52:49.485 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:52:49.561 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/05/06 19:52:49.569 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:52:49.581 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:52:49.660 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:52:49.681 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:52:49.766 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:52:49.767 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:52:49.769 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:52:49.859 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:52:49.970 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:52:49.978 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:52:50.070 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:52:50.071 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:52:50.160 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:52:50.282 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/06 19:52:50.359 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:52:50.367 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] check diff failed 5-th time, retry later table resourcecontrol.finish_mark exists check diff successfully [2024/05/06 19:52:50.660 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/06 19:52:50.678 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:52:50.765 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:52:50.774 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:52:50.779 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:52:50.871 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:52:50.873 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:52:50.877 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:52:50.960 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36ff4880001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h, pid:6607, start at 2024-05-06 19:52:47.95263327 +0800 CST m=+5.400571975 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:54:47.959 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:52:47.954 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:42:47.954 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36ff4880001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h, pid:6607, start at 2024-05-06 19:52:47.95263327 +0800 CST m=+5.400571975 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:54:47.959 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:52:47.954 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:42:47.954 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36ff4a0c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h, pid:6689, start at 2024-05-06 19:52:48.012815828 +0800 CST m=+5.410149133 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:54:48.018 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:52:48.003 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:42:48.003 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 27-th check, retry later [2024/05/06 19:52:50.975 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:52:51.062 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/05/06 19:52:51.072 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo + source /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status_with_redo + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo The 1 times to try to start tidb cluster... [2024/05/06 19:52:51.181 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:52:51.277 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:52:51.358 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:52:51.365 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:52:51.366 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:52:51.579 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/06 19:52:51.668 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:52:51.677 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] wait process cdc.test exit for 2-th time... [2024/05/06 19:52:51.873 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:52:52 CST 2024] <<<<<< run test case resourcecontrol success! >>>>>> [2024/05/06 19:52:52.057 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:52:52.062 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:52:52.063 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:52:52.070 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:52:52.073 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:52:52.081 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:52:52.166 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:52:52.174 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:52:52.271 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:52:52.362 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:52:52.367 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:52:52.480 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/05/06 19:52:52.484 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [2024/05/06 19:52:52.557 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/05/06 19:52:52.561 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/05/06 19:52:52.563 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [Mon May 6 19:52:52 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.79177919.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully [2024/05/06 19:52:52.781 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/05/06 19:52:52.969 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:52:52.980 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:52:53.065 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:52:53.076 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:52:53.076 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:52:53.192 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:52:53.193 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:52:53.361 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] [2024/05/06 19:52:53.496 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] table ddl_manager.finish_mark not exists for 28-th check, retry later [2024/05/06 19:52:53.559 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/05/06 19:52:53.570 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/05/06 19:52:53.588 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:52:53.668 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:52:53.668 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:52:53.671 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:52:53.776 +08:00] [INFO] [main.go:178] ["1 insert success: 1600"] [2024/05/06 19:52:53.960 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs085ea4bb_d57f_4a86_9f7c_e70104dc572c"] [2024/05/06 19:52:53.977 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 19:52:54.068 +08:00] [INFO] [main.go:178] ["0 insert success: 1200"] [2024/05/06 19:52:54.074 +08:00] [INFO] [main.go:199] ["0 delete success: 600"] [2024/05/06 19:52:54.088 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs0a3aec31_07ad_4e14_9884_9fc5c1998d2a"] [2024/05/06 19:52:54.168 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs5dd9413d_009d_4dea_9353_671ad70d4129"] [2024/05/06 19:52:54.197 +08:00] [INFO] [main.go:178] ["1 insert success: 1700"] [2024/05/06 19:52:54.264 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs0008c2f7_2b9a_4b1f_8ca2_2d63be899043"] check diff failed 1-th time, retry later [2024/05/06 19:52:54.387 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:52:54.389 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:52:54.390 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb9d18db3_882f_4d27_9292_d3631449074e"] [2024/05/06 19:52:54.483 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsfa598277_4026_4d88_8bd4_f56589806e9a"] start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... [2024/05/06 19:52:54.584 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs8ad44f19_be7a_430a_95e6_bb222512d50b"] [2024/05/06 19:52:54.662 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:52:54.665 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:52:54.866 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:52:54.866 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:52:54.966 +08:00] [INFO] [main.go:178] ["0 insert success: 1300"] [2024/05/06 19:52:54.971 +08:00] [INFO] [main.go:178] ["1 insert success: 1800"] [2024/05/06 19:52:55.075 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:52:55.160 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:52:55.164 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs937f0222_0d46_48ed_9fe9_a9c1ccb6583c"] table ddl_manager.finish_mark not exists for 29-th check, retry later [2024/05/06 19:52:55.461 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:52:55.475 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:52:55.480 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:52:55.487 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:52:55.579 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:52:55.661 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:52:55.758 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:52:55.760 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:52:55.764 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:52:55.769 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:52:55.962 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:52:55.975 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:52:55.985 +08:00] [INFO] [main.go:178] ["1 insert success: 1900"] [2024/05/06 19:52:56.074 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:52:56.075 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:52:56.087 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:52:55 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4bfb9d25-c65a-41e0-8d12-f9c34204d8d5 {"id":"4bfb9d25-c65a-41e0-8d12-f9c34204d8d5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996373} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbfa75633 4bfb9d25-c65a-41e0-8d12-f9c34204d8d5 /tidb/cdc/default/default/upstream/7365853277194306638 {"id":7365853277194306638,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4bfb9d25-c65a-41e0-8d12-f9c34204d8d5 {"id":"4bfb9d25-c65a-41e0-8d12-f9c34204d8d5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996373} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbfa75633 4bfb9d25-c65a-41e0-8d12-f9c34204d8d5 /tidb/cdc/default/default/upstream/7365853277194306638 {"id":7365853277194306638,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4bfb9d25-c65a-41e0-8d12-f9c34204d8d5 {"id":"4bfb9d25-c65a-41e0-8d12-f9c34204d8d5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996373} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbfa75633 4bfb9d25-c65a-41e0-8d12-f9c34204d8d5 /tidb/cdc/default/default/upstream/7365853277194306638 {"id":7365853277194306638,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.7985.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-only-block-related-table check diff failed 2-th time, retry later [2024/05/06 19:52:56.174 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7365853277194306638,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-06T19:52:56.392703665+08:00","start_ts":449576010069835778,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576010069835778,"checkpoint_ts":449576010069835778,"checkpoint_time":"2024-05-06 19:52:56.304"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:52:56.365 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:52:56.374 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:52:56.463 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:52:56.465 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:52:56.470 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:52:56.571 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:52:56.572 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:52:56.575 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:52:56.665 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:52:56.669 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:52:56.768 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:52:56.860 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 19:52:56.957 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:52:56.960 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:52:56.961 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:52:56.978 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] table ddl_manager.finish_mark not exists for 30-th check, retry later [2024/05/06 19:52:57.258 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:52:57.368 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:52:57.560 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:52:57.659 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:52:57.667 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:52:58.063 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:52:58.081 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:52:58.162 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] + set +x [2024/05/06 19:52:58.462 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:52:58.566 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:52:58.671 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] check diff failed 3-th time, retry later [2024/05/06 19:52:58.874 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 31-th check, retry later [2024/05/06 19:52:59.273 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:52:59.360 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:52:59.360 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:52:59.564 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:52:59.673 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:52:59.867 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:52:59.966 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:52:59.968 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:00.065 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:00.164 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:53:00.171 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:00.172 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:00.361 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:53:00.372 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:00.376 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:00.473 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] check diff failed 4-th time, retry later [2024/05/06 19:53:00.575 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:00.578 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:00.589 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:00.762 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:53:00.766 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:00.778 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:53:00.873 +08:00] [INFO] [main.go:88] ["testGetDefaultValue take 24.173798649s"] [2024/05/06 19:53:01.068 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:01.070 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:01.070 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] table ddl_manager.finish_mark not exists for 32-th check, retry later [2024/05/06 19:53:01.277 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:01.359 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:01.362 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:01.481 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:01.483 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:01.566 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:01.675 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:01.682 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:01.686 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:01.777 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:01.779 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:01.788 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:01.869 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:01.963 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:02.065 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:02.068 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:02.173 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:02.272 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] check diff failed 5-th time, retry later [2024/05/06 19:53:02.469 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:02.581 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later [2024/05/06 19:53:02.658 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:02.673 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:02.783 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:02.861 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:02.862 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:53:02.958 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:02.960 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:02.965 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:03.075 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:03.077 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:03.177 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:03.266 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:03.273 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:03.365 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] table ddl_manager.finish_mark not exists for 33-th check, retry later [2024/05/06 19:53:03.577 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:03.857 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:03.873 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:03.962 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:03.983 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:04.075 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:04.084 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:04.084 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:04.165 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:04.167 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:04.273 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:04.284 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:04.369 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:53:04.384 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:04.467 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:04.474 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:53:04.678 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:04.878 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] check diff failed 6-th time, retry later [2024/05/06 19:53:05.160 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:05.168 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:53:05.169 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:05.263 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:05.279 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:53:05.363 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:05.366 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:53:05.383 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] table ddl_manager.finish_mark not exists for 34-th check, retry later [2024/05/06 19:53:05.478 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:05.560 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:05.571 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:53:05.581 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:53:05.676 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:05.677 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:53:05.779 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:53:05.959 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:53:06.172 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:53:06.173 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:53:06.265 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:53:06.359 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:06.367 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:53:06.460 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] table ddl_only_block_related_table.finish_mark exists [2024/05/06 19:53:06.481 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:53:06.572 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:53:06.586 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:53:06.661 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:53:06.680 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] \033[0;36m<<< Run all test success >>>\033[0m [2024/05/06 19:53:06.761 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:53:06.776 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:53:06.785 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:53:06.874 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:53:06.977 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] check diff successfully wait process 7922 exit for 1-th time... [2024/05/06 19:53:07.082 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:53:07.162 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:53:07.164 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370041d00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:10188, start at 2024-05-06 19:53:03.906965344 +0800 CST m=+5.247291500 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:03.915 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:03.911 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:03.911 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370041d00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:10188, start at 2024-05-06 19:53:03.906965344 +0800 CST m=+5.247291500 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:03.915 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:03.911 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:03.911 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370043400015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:10267, start at 2024-05-06 19:53:03.985426999 +0800 CST m=+5.276514700 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:03.995 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:03.952 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:03.952 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 35-th check, retry later [2024/05/06 19:53:07.272 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:53:07.460 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] wait process 7922 exit for 2-th time... [2024/05/06 19:53:07.565 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:53:07.660 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:53:07.673 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:53:07.765 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:53:07.877 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:53:07.965 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] wait process 7922 exit for 3-th time... [2024/05/06 19:53:08.069 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:53:08.176 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:53:08.185 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] check diff failed 1-th time, retry later /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (7922) - No such process wait process 7922 exit for 4-th time... process 7922 already exit [Mon May 6 19:53:08 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.80898091.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL [2024/05/06 19:53:08.268 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:53:08.293 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:53:08.388 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:53:08.391 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:53:08.460 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:53:08.578 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:53:08.667 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:53:08.862 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:53:08.971 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:53:08.986 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11531.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 36-th check, retry later [2024/05/06 19:53:09.073 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:53:09.100 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:53:09.197 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:53:09.204 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:53:09.284 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:53:09.292 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:53:09.381 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:53:09.460 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:53:09.504 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:53:09.779 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:53:09.865 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs1adb2f96_b583_47d0_b3e7_1b421a029302"] [2024/05/06 19:53:10.085 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:53:10.086 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLscbfb9562_e6bf_4192_a5c2_7214c18804e6"] [2024/05/06 19:53:10.101 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd0be22c8_642b_443e_9528_42164aed4cd7"] [2024/05/06 19:53:10.281 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs391799e8_d36b_49bd_8580_6df23259e8bf"] [2024/05/06 19:53:10.561 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:53:10.567 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc0c02206_a09f_4475_9c6b_183575251037"] [2024/05/06 19:53:10.569 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:53:10.661 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs7d3bc267_6b44_42c5_9566_31ce9726d147"] [2024/05/06 19:53:10.662 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs3c4a10de_34cd_4bd3_92c8_640f9f709f7d"] check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 37-th check, retry later + set +x + tso='449576013466173441 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576013466173441 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449576013466173441 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Mon May 6 19:53:10 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1156411566.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:53:11.262 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:53:11.268 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:53:11.274 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:53:11.280 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:53:11.476 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:53:11.664 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:53:11.759 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:53:11.772 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:53:11.861 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:53:11.864 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:53:11.865 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:53:11.872 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:53:11.873 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:53:11.874 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:53:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365853277194306638 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 19:52:56.392703665 +0800 CST StartTs:449576010069835778 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00286b5f0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576010082942983} {CheckpointTs:449576013818757127 MinTableBarrierTs:449576013818757127 AdminJobType:noop} span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449576013818757128, checkpointTs: 449576013818757127, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576013818757128, checkpointTs: 449576013818757127, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449576013818757128, checkpointTs: 449576013818757127, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576013818757128, checkpointTs: 449576013818757127, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc040db9-5a23-4d83-b625-737c57651de9 {"id":"cc040db9-5a23-4d83-b625-737c57651de9","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996388} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbfa756ff cc040db9-5a23-4d83-b625-737c57651de9 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365853277194306638,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T19:52:56.392703665+08:00","start-ts":449576010069835778,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576010082942983} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449576013818757127,"min-table-barrier-ts":449576013818757127,"admin-job-type":0} /tidb/cdc/default/default/task/position/cc040db9-5a23-4d83-b625-737c57651de9/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853277194306638 {"id":7365853277194306638,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365853277194306638 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 19:52:56.392703665 +0800 CST StartTs:449576010069835778 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00286b5f0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576010082942983} {CheckpointTs:449576013818757127 MinTableBarrierTs:449576013818757127 AdminJobType:noop} span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449576013818757128, checkpointTs: 449576013818757127, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576013818757128, checkpointTs: 449576013818757127, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449576013818757128, checkpointTs: 449576013818757127, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576013818757128, checkpointTs: 449576013818757127, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc040db9-5a23-4d83-b625-737c57651de9 {"id":"cc040db9-5a23-4d83-b625-737c57651de9","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996388} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbfa756ff cc040db9-5a23-4d83-b625-737c57651de9 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365853277194306638,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T19:52:56.392703665+08:00","start-ts":449576010069835778,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576010082942983} /tidb/cdc/default/default/changefeed/status/ddl-only-blo+ grep -q 'failed to get info:' ck-related-table {"checkpoint-ts":449576013818757127,"min-table-barrier-ts":449576013818757127,"admin-job-type":0} /tidb/cdc/default/default/task/position/cc040db9-5a23-4d83-b625-737c57651de9/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853277194306638 {"id":7365853277194306638,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365853277194306638 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 19:52:56.392703665 +0800 CST StartTs:449576010069835778 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00286b5f0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576010082942983} {CheckpointTs:449576013818757127 MinTableBarrierTs:449576013818757127 AdminJobType:noop} span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449576013818757128, checkpointTs: 449576013818757127, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576013818757128, checkpointTs: 449576013818757127, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449576013818757128, checkpointTs: 449576013818757127, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576013818757128, checkpointTs: 449576013818757127, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc040db9-5a23-4d83-b625-737c57651de9 {"id":"cc040db9-5a23-4d83-b625-737c57651de9","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996388} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbfa756ff cc040db9-5a23-4d83-b625-737c57651de9 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365853277194306638,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T19:52:56.392703665+08:00","start-ts":449576010069835778,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576010082942983} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449576013818757127,"min-table-barrier-ts":449576013818757127,"admin-job-type":0} /tidb/cdc/default/default/task/position/cc040db9-5a23-4d83-b625-737c57651de9/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853277194306638 {"id":7365853277194306638,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later check diff failed 3-th time, retry later [2024/05/06 19:53:12.468 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:53:12.470 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:53:12.475 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:53:12.561 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:53:12.666 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:53:12.770 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:53:12.775 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:53:12.868 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:53:12.871 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:53:12.873 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs2c3b3797_fc4b_4bf6_9baa_6f35050db43f"] [2024/05/06 19:53:12.874 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:53:12.874 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:53:12.960 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:53:12.964 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:53:12.970 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] table ddl_manager.finish_mark not exists for 38-th check, retry later [2024/05/06 19:53:13.672 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:53:13.763 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:53:13.763 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:53:13.773 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:53:13.861 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:53:13.967 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:53:13.978 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 20.30 secs (180724899 bytes/sec) [Pipeline] { [Pipeline] cache [2024/05/06 19:53:14.066 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:53:14.073 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:53:14.076 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:53:14.163 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:53:14.168 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:53:14.173 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:14.258 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:53:14.265 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:53:14.269 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:53:13 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f0f7f054-54cd-4630-897e-1710f5c8b078 {"id":"f0f7f054-54cd-4630-897e-1710f5c8b078","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996391} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbfea273b f0f7f054-54cd-4630-897e-1710f5c8b078 /tidb/cdc/default/default/upstream/7365853346796989907 {"id":7365853346796989907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f0f7f054-54cd-4630-897e-1710f5c8b078 {"id":"f0f7f054-54cd-4630-897e-1710f5c8b078","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996391} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbfea273b f0f7f054-54cd-4630-897e-1710f5c8b078 /tidb/cdc/default/default/upstream/7365853346796989907 {"id":7365853346796989907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f0f7f054-54cd-4630-897e-1710f5c8b078 {"id":"f0f7f054-54cd-4630-897e-1710f5c8b078","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996391} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbfea273b f0f7f054-54cd-4630-897e-1710f5c8b078 /tidb/cdc/default/default/upstream/7365853346796989907 {"id":7365853346796989907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449576013466173441 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11629.out cli changefeed create --start-ts=449576013466173441 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365853346796989907,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:53:14.277761601+08:00","start_ts":449576013466173441,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576013466173441,"checkpoint_ts":449576013466173441,"checkpoint_time":"2024-05-06 19:53:09.260"} PASS coverage: 2.7% of statements in github.com/pingcap/tiflow/... check diff failed 4-th time, retry later check diff successfully check_ts_not_forward ddl-only-block-related-table [2024/05/06 19:53:14.971 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:53:14.981 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:14.984 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:15.078 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:53:15.088 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:53:15.179 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:15.262 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:15.268 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:53:15.273 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:53:15.276 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:15.277 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:53:15.286 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:53:15.287 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:15.359 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:53:15.367 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:15.367 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] table ddl_manager.finish_mark not exists for 39-th check, retry later + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 4825 0 --:--:-- --:--:-- --:--:-- 4911 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 19:53:09.260","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-06 19:53:15.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:53:09.260","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:53:15.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:53:09.260","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:53:15.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-06 19:53:09.260' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:53:09.260","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:53:15.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:53:09.260","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:53:15.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-06 19:53:15' + echo 'sink_checkpoint_ts is 2024-05-06' 19:53:09.260 sink_checkpoint_ts is 2024-05-06 19:53:09.260 ++ date -d '2024-05-06 19:53:09.260' +%s + checkpoint_timestamp=1714996389 ++ date -d '2024-05-06 19:53:15' +%s + current_timestamp=1714996395 + '[' 6 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' [2024/05/06 19:53:16.168 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:16.174 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:16.180 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:16.359 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:16.361 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:16.380 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later [2024/05/06 19:53:16.459 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:16.466 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:16.469 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:16.475 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:53:16.559 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:16.561 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:53:16.565 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:16.567 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:16.570 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:16.576 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] ____________________________________ *************************** 1. row *************************** primary_ts: 449575951753543681 secondary_ts: 449575954476957709 *************************** 2. row *************************** primary_ts: 449575959617863680 secondary_ts: 449575959758897153 *************************** 3. row *************************** primary_ts: 449575967482183680 secondary_ts: 449575967597264903 *************************** 4. row *************************** primary_ts: 449575975346503680 secondary_ts: 449575975527120897 *************************** 5. row *************************** primary_ts: 449575983210823680 secondary_ts: 449575983364964353 *************************** 6. row *************************** primary_ts: 449575991075143680 secondary_ts: 449575991229546497 *************************** 7. row *************************** primary_ts: 449575998939463680 secondary_ts: 449575999106973697 *************************** 8. row *************************** primary_ts: 449576006803783680 secondary_ts: 449576006971293697 *************************** 9. row *************************** primary_ts: 449576014668103680 secondary_ts: 449576014861828097 skip invalid syncpoint primary_ts: 449575951753543681, first_ddl_ts: 449575955069665307 check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully [2024/05/06 19:53:13.854 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/06 19:53:13.854 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7365852703802375758] [2024/05/06 19:53:13.855 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 19:53:13.855 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/06 19:53:13.855 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/06 19:53:13.855 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 19:53:13.856 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/06 19:53:13.856 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7365852703802375758] [2024/05/06 19:53:13.856 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 19:53:13.856 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/06 19:53:13.857 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/06 19:53:13.857 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 19:53:13.857 +08:00] [INFO] [tikv_driver.go:197] ["using API V1."] [2024/05/06 19:53:13.858 +08:00] [INFO] [main.go:180] ["genLock started"] [2024/05/06 19:53:13.858 +08:00] [INFO] [store_cache.go:477] ["change store resolve state"] [store=1] [addr=127.0.0.1:20160] [from=unresolved] [to=resolved] [liveness-state=reachable] wait process cdc.test exit for 1-th time... check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 40-th check, retry later wait process cdc.test exit for 2-th time... [2024/05/06 19:53:17.383 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:17.464 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:17.471 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:17.566 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:17.569 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:17.583 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:17.587 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:17.673 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:17.674 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:17.677 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:17.679 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:17.681 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:53:17.686 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:17.687 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:17.758 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:17.760 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:53:18 CST 2024] <<<<<< run test case syncpoint success! >>>>>> table test.t1 exists + sleep 5 [2024/05/06 19:53:18.561 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:18.573 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:18.580 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:18.680 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:18.680 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:18.764 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:18.769 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:18.860 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:18.862 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:18.865 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:18.868 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:18.873 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:18.877 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:18.961 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:18.963 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:18.966 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] check diff successfully table ddl_manager.finish_mark not exists for 41-th check, retry later [2024/05/06 19:53:19.868 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:19.970 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:19.974 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:20.068 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:20.076 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:20.078 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:20.081 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:20.164 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:20.165 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:20.170 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:20.176 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:53:20.177 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:20.180 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:20.181 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:20.187 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:20.190 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] check diff failed 1-th time, retry later [2024/05/06 19:53:20.981 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:21.075 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:21.079 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:53:21.174 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] table ddl_manager.finish_mark not exists for 42-th check, retry later [2024/05/06 19:53:21.265 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:21.365 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:53:21.371 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:53:21.465 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:21.469 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:53:21.563 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:21.565 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:21.571 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:53:21.578 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:53:21.660 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:53:21.660 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:21.661 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] check diff failed 2-th time, retry later [2024/05/06 19:53:22.664 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:53:22.858 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:53:22.869 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:53:22.879 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:53:22.971 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:53:23.059 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:23.069 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:53:23.071 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:53:23.175 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:53:23.176 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:53:23.272 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:53:23.274 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:53:23.274 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:23.364 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:53:23.365 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:53:23.374 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] Cache restored successfully (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc) 3667975168 bytes in 7.25 secs (505642182 bytes/sec) [Pipeline] { [Pipeline] sh table ddl_manager.finish_mark not exists for 43-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 3555 0 --:--:-- --:--:-- --:--:-- 3573 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:53:21.711","puller_resolved_ts":"2024-05-06 19:53:15.910","last_synced_ts":"2024-05-06 19:53:16.410","now_ts":"2024-05-06 19:53:23.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:53:21.711","puller_resolved_ts":"2024-05-06' '19:53:15.910","last_synced_ts":"2024-05-06' '19:53:16.410","now_ts":"2024-05-06' '19:53:23.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:53:21.711","puller_resolved_ts":"2024-05-06' '19:53:15.910","last_synced_ts":"2024-05-06' '19:53:16.410","now_ts":"2024-05-06' '19:53:23.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 [Pipeline] sh [2024/05/06 19:53:23.960 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:53:24.078 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:53:24.085 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G12 Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=9228ec65-a589-46b5-a263-b801b5f6b6ad BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-54cks HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-gl74w GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-gl74w cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-54cks GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-54cks GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh [2024/05/06 19:53:24.162 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:53:24.165 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:53:24.166 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:24.190 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:53:24.193 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:53:24.195 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:53:24.263 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:53:24.285 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:53:24.360 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:53:24.362 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:53:24.370 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G15 Run cases: new_ci_collation batch_add_table multi_rocks PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=97454d61-1d44-439a-91e1-c2d4df14c3de BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-rqt30 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-m70kp GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-rqt30 cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-m70kp GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-rqt30 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh check diff failed 3-th time, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G17 Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=3f3889a0-d8c6-4db6-b2b2-060e9d1467e8 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-89kh8 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-w3fs6 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-w3fs6 cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-89kh8 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-89kh8 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh [2024/05/06 19:53:23.874 +08:00] [INFO] [main.go:196] ["genLock done"] [2024/05/06 19:53:23.875 +08:00] [INFO] [resource_manager_client.go:295] ["[resource manager] exit resource token dispatcher"] [2024/05/06 19:53:23.875 +08:00] [INFO] [tso_dispatcher.go:268] ["exit tso dispatcher loop"] [2024/05/06 19:53:23.875 +08:00] [INFO] [tso_dispatcher.go:214] ["exit tso requests cancel loop"] [2024/05/06 19:53:23.875 +08:00] [INFO] [pd_service_discovery.go:550] ["[pd] exit member loop due to context canceled"] [2024/05/06 19:53:23.875 +08:00] [INFO] [tso_client.go:140] ["closing tso client"] [2024/05/06 19:53:23.875 +08:00] [INFO] [tso_dispatcher.go:455] ["[tso] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2024/05/06 19:53:23.875 +08:00] [INFO] [tso_dispatcher.go:380] ["[tso] exit tso dispatcher"] [dc-location=global] [2024/05/06 19:53:23.875 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/06 19:53:23.875 +08:00] [INFO] [tso_client.go:145] ["close tso client"] [2024/05/06 19:53:23.875 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/06 19:53:23.875 +08:00] [INFO] [tso_client.go:155] ["tso client is closed"] [2024/05/06 19:53:23.875 +08:00] [INFO] [pd_service_discovery.go:637] ["[pd] close pd service discovery client"] + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G19 Run cases: changefeed_fast_fail batch_update_to_no_batch changefeed_resume_with_checkpoint_ts PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=cf23b7a4-c894-4fe5-abbe-cee018b0aacb BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-5k7gg HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G19 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-l3m7x GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-5k7gg cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-l3m7x GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-5k7gg GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_fast_fail/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh [2024/05/06 19:53:24.973 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:53:25.172 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:53:25.257 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:53:25.270 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:53:25.273 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:53:25.279 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:53:25.288 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:53:25.290 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G21 Run cases: bank kill_owner_with_ddl owner_remove_table_error PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=d9ee082e-310d-46c1-b739-30f9e2be2686 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-59v7t HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G21 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-5t6wm GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-59v7t cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-5t6wm GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-59v7t GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/bank/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 44-th check, retry later [2024/05/06 19:53:25.364 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:53:25.372 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:53:25.374 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:53:25.375 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:53:25.385 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G16 Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=94c048bc-9bd4-4c7f-80a0-2f1d9ade504f BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-dwwvr HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-j5rm5 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-j5rm5 cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-dwwvr GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-dwwvr GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh [2024/05/06 19:53:26.065 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:53:26.075 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G20 Run cases: tidb_mysql_test ddl_reentrant multi_cdc_cluster PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=af27ac17-5654-4892-bf4e-17c9b1e38087 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-b2lwz HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G20 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-p11tp GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-b2lwz cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-p11tp GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-b2lwz GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/tidb_mysql_test/run.sh using Sink-Type: mysql... <<================= [2024/05/06 19:53:26.175 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:53:26.176 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:53:26.176 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:53:26.182 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:53:26.275 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs9e9692ac_3c2e_405e_b75b_358266f65f4f"] [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G18 Run cases: availability http_proxies sequence PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=7977eced-36df-4fca-a0c9-b728530c56f0 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-c3sn1 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G18 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-cdqw4 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-c3sn1 cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-cdqw4 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-c3sn1 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/availability/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh check diff failed 4-th time, retry later [2024/05/06 19:53:26.672 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs58a54c25_ce44_48a8_abe0_14c52a9e6bca"] [2024/05/06 19:53:26.873 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs397459bd_2b4d_4e5e_9cc6_4b43ff338b45"] [2024/05/06 19:53:26.882 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs5a9b0e35_21c0_423f_b877_f0b762cbe60b"] + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G13 [2024/05/06 19:53:26.992 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:53:26.994 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:53:27.071 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs1e35b203_9953_4717_9074_78c54f3d8015"] + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G14 Run cases: changefeed_finish force_replicate_table PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=42dcf241-ec74-4ff8-ba3b-250fc2310bb0 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-98cvh HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-57vfd GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-57vfd cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-98cvh GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-98cvh GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 45-th check, retry later [2024/05/06 19:53:27.360 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:53:27.377 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] Run cases: tiflash region_merge common_1 PROW_JOB_ID=a9242cee-122c-4f99-9a86-3cf35b21c615 JENKINS_NODE_COOKIE=a2267054-3c3e-4745-96c3-dc74708ce8d3 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/ GOLANG_VERSION=1.21.6 HOSTNAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-fd5nv HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test","buildid":"1787445299229233157","prowjobid":"a9242cee-122c-4f99-9a86-3cf35b21c615","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-8.1","base_sha":"38eb040812f817df410abe870952629df60fd01f","base_link":"https://github.com/pingcap/tiflow/commit/38eb040812f817df410abe870952629df60fd01f","pulls":[{"number":11026,"author":"ti-chi-bot","sha":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","title":"kafka(ticdc): Fix java example license issue (#11024)","link":"https://github.com/pingcap/tiflow/pull/11026","commit_link":"https://github.com/pingcap/tiflow/pull/11026/commits/e1a27adc69acc9d50db0c3fea2945bbd6d983e30","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#77 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787445299229233157 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=a0a6bb5614d55afbab88f0d1ae459423175a5407 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/77/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-821f0 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a975a8e7c9a45cad2cd9ab0afcefd3e1fdc24d24ceebc1e869b7c4e4d0ed24d4 NODE_LABELS=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-fd5nv cap_tiflow_release-8_1_pull_cdc_integration_mysql_test_77-821f0 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-fd5nv GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-8.1/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=77 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 19:53:27.488 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs25a3828d_d81f_4e23_a344_9cbb4f910e5b"] [2024/05/06 19:53:27.495 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:53:27.497 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs0a296680_94db_4eef_a044_87f87fd0b582"] [2024/05/06 19:53:27.564 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:53:27.575 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:53:27.577 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:53:27.668 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc277417e_8dc7_4b30_8b3d_842430238171"] start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... [2024/05/06 19:53:27.770 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:53:27.874 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:53:27.960 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:53:27.975 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_ddl/sync_diff/output" source-instances = ["tidb"] target-instance = "mysql" target-check-tables = ["consistent_replicate_ddl.usertable*","consistent_replicate_ddl.check*"] [data-sources] [data-sources.tidb] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "449575982948417570" [data-sources.mysql] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/05/06 19:53:15.613 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 19:53:15.614 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2024/05/06 19:53:15.614 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=:6060] [2024/05/06 19:53:15.614 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 19:53:15.615 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449575981296910346] [resolvedTs=449575982948417570] [2024/05/06 19:53:15.616 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 19:53:15.616 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 19:53:15.616 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/06 19:53:15.616 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=1.543404ms] [2024/05/06 19:53:15.616 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=1.679451ms] [2024/05/06 19:53:15.620 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 19:53:15.623 +08:00] [INFO] [mysql.go:197] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 19:53:15.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:53:15.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:53:15.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:53:15.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:53:15.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:53:15.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:53:15.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:53:15.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:53:15.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:53:15.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:53:15.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:53:15.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:53:15.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:53:15.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:53:15.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:53:15.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:53:15.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 19:53:15.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 19:53:15.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 19:53:15.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 19:53:15.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 19:53:15.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 19:53:15.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 19:53:15.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 19:53:15.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 19:53:15.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 19:53:15.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 19:53:15.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 19:53:15.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 19:53:15.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 19:53:15.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 19:53:15.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 19:53:15.624 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 19:53:15.624 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 19:53:15.624 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/06 19:53:15.628 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 19:53:15.630 +08:00] [INFO] [mysql_ddl_sink.go:105] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/06 19:53:15.630 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449575981296910342,\"CommitTs\":449575981296910346,\"Query\":\"DROP TABLE `consistent_replicate_ddl`.`usertable1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable1\",\"TableID\":108,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":4,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 19:53:15.631 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="DROP TABLE `consistent_replicate_ddl`.`usertable1`"] [commitTs=449575981296910346] [namespace=] [changefeed=] [2024/05/06 19:53:15.833 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="DROP TABLE `consistent_replicate_ddl`.`usertable1`"] [duration=202.579174ms] [namespace=] [changefeed=] [2024/05/06 19:53:15.833 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449575981414875144,\"CommitTs\":449575981414875147,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable1\",\"TableID\":114,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 19:53:15.833 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1"] [commitTs=449575981414875147] [namespace=] [changefeed=] [2024/05/06 19:53:15.968 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1"] [duration=134.439308ms] [namespace=] [changefeed=] [2024/05/06 19:53:15.968 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449575981559054345,\"CommitTs\":449575981559054349,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable1_1\",\"TableID\":114,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 19:53:16.169 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1"] [commitTs=449575981559054349] [namespace=] [changefeed=] [2024/05/06 19:53:16.308 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1"] [duration=139.466576ms] [namespace=] [changefeed=] [2024/05/06 19:53:16.308 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449575981690126361,\"CommitTs\":449575981690126364,\"Query\":\"CREATE DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl1\",\"Table\":\"\",\"TableID\":0,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":1,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 19:53:16.509 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="CREATE DATABASE consistent_replicate_ddl1"] [commitTs=449575981690126364] [namespace=] [changefeed=] [2024/05/06 19:53:16.589 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="CREATE DATABASE consistent_replicate_ddl1"] [duration=80.282357ms] [namespace=] [changefeed=] [2024/05/06 19:53:16.589 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449575981834305542,\"CommitTs\":449575981834305545,\"Query\":\"ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"\",\"TableID\":0,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":26,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 19:53:16.589 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [commitTs=449575981834305545] [namespace=] [changefeed=] [2024/05/06 19:53:16.663 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [duration=73.929014ms] [namespace=] [changefeed=] [2024/05/06 19:53:16.663 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449575981965639705,\"CommitTs\":449575981965639708,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 19:53:16.664 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [commitTs=449575981965639708] [namespace=] [changefeed=] [2024/05/06 19:53:16.740 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [duration=76.546511ms] [namespace=] [changefeed=] [2024/05/06 19:53:16.740 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449575982161985543,\"CommitTs\":449575982161985547,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":6,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 19:53:16.941 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0"] [commitTs=449575982161985547] [namespace=] [changefeed=] [2024/05/06 19:53:17.205 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0"] [duration=264.113582ms] [namespace=] [changefeed=] [2024/05/06 19:53:17.205 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449575982306164742,\"CommitTs\":449575982306164747,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":5,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 19:53:17.205 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)"] [commitTs=449575982306164747] [namespace=] [changefeed=] [2024/05/06 19:53:17.464 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)"] [duration=258.605329ms] [namespace=] [changefeed=] [2024/05/06 19:53:17.464 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449575982424391687,\"CommitTs\":449575982424391690,\"Query\":\"DROP DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl1\",\"Table\":\"\",\"TableID\":0,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":2,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 19:53:17.464 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="DROP DATABASE consistent_replicate_ddl1"] [commitTs=449575982424391690] [namespace=] [changefeed=] [2024/05/06 19:53:17.657 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="DROP DATABASE consistent_replicate_ddl1"] [duration=193.058706ms] [namespace=] [changefeed=] [2024/05/06 19:53:17.657 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449575982528987158,\"CommitTs\":449575982528987163,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable3\",\"TableID\":112,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 19:53:17.657 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"] [commitTs=449575982528987163] [namespace=] [changefeed=] [2024/05/06 19:53:17.732 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"] [duration=74.853389ms] [namespace=] [changefeed=] [2024/05/06 19:53:17.732 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449575982686273543,\"CommitTs\":449575982686273546,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2_1\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 19:53:17.732 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`"] [commitTs=449575982686273546] [namespace=] [changefeed=] [2024/05/06 19:53:17.869 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`"] [duration=136.846585ms] [namespace=] [changefeed=] [2024/05/06 19:53:17.869 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449575982686273543,\"CommitTs\":449575982686273546,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable3_1\",\"TableID\":112,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 19:53:17.869 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`"] [commitTs=449575982686273546] [namespace=] [changefeed=] [2024/05/06 19:53:18.008 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`"] [duration=139.140228ms] [namespace=] [changefeed=] [2024/05/06 19:53:18.009 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449575982804500505,\"CommitTs\":449575982804500509,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable3_1\",\"TableID\":112,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":12,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 19:53:18.009 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)"] [commitTs=449575982804500509] [namespace=] [changefeed=] [2024/05/06 19:53:18.085 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)"] [duration=76.501015ms] [namespace=] [changefeed=] [2024/05/06 19:53:18.085 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449575982948417566,\"CommitTs\":449575982948417570,\"Query\":\"CREATE table consistent_replicate_ddl.check1(id int primary key)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"check1\",\"TableID\":129,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 19:53:18.286 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="CREATE table consistent_replicate_ddl.check1(id int primary key)"] [commitTs=449575982948417570] [namespace=] [changefeed=] [2024/05/06 19:53:18.368 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="CREATE table consistent_replicate_ddl.check1(id int primary key)"] [duration=81.507091ms] [namespace=] [changefeed=] [2024/05/06 19:53:18.368 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449575982948417570] [2024/05/06 19:53:18.368 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449575982948417570] [2024/05/06 19:53:18.468 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}] [checkpointTs=449575982948417570] [2024/05/06 19:53:18.468 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}] [checkpointTs=449575982948417570] [2024/05/06 19:53:18.568 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}] [checkpointTs=449575982948417570] [2024/05/06 19:53:18.569 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}] [checkpointTs=449575982948417570] [2024/05/06 19:53:18.669 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=101] [appliedDDLCount=14] [currentCheckpoint=449575982948417570] [2024/05/06 19:53:18.669 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 19:53:18.669 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/06 19:53:18.669 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 19:53:18.669 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 19:53:18.669 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/06 19:53:18.669 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/06 19:53:18.669 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 19:53:18.669 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 19:53:18.669 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 19:53:18.669 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 19:53:18.669 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 19:53:18.669 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 19:53:18.669 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/06 19:53:18.669 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 19:53:18.669 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/06 19:53:18.669 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] Apply redo log successfully table consistent_replicate_ddl.check1 exists check diff successfully [Mon May 6 19:53:18 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> count(*) 50 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:53:27 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/changefeed_fast_fail Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... [2024/05/06 19:53:28.465 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:53:28.473 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] Release Version: v8.1.0-alpha-20-g13eaf8aaa [2024/05/06 19:53:28.660 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:53:28.667 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] start tidb cluster in /tmp/tidb_cdc_test/bank [2024/05/06 19:53:28.669 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] You are running an older version of MinIO released 3 years ago [2024/05/06 19:53:28.671 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] Edition: Community [2024/05/06 19:53:28.764 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d [2024/05/06 19:53:28.765 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] Git Branch: release-8.1 [2024/05/06 19:53:28.766 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] Update: Run `mc admin update` [2024/05/06 19:53:28.769 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] Starting Upstream PD... [2024/05/06 19:53:28.777 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:53:28.778 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Attempting encryption of all config, IAM users and policies on MinIO backend Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... start tidb cluster in /tmp/tidb_cdc_test/clustered_index Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... [2024/05/06 19:53:28.880 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Endpoint: http://127.0.0.1:24927 Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 [2024/05/06 19:53:28.972 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] UTC Build Time: 2024-04-30 07:22:55 [2024/05/06 19:53:29.072 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] Object API (Amazon S3 compatible): Verifying upstream PD is started... Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 46-th check, retry later [2024/05/06 19:53:29.164 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... [2024/05/06 19:53:29.684 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:53:29.688 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:53:29.859 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:53:29.898 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:53:29.901 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:53:29.902 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:53:29.905 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:53:29.906 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:53:29.964 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:53:29.967 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:53:29.984 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:53:29.994 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] table test.t2 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release The 1 times to try to start tidb cluster... [2024/05/06 19:53:30.263 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:30.274 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCE4260A19A7FD < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 11:53:30 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... [2024/05/06 19:53:30.464 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:53:30.474 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/availability Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... check diff failed 6-th time, retry later table ddl_manager.finish_mark not exists for 47-th check, retry later [2024/05/06 19:53:31.094 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:53:31.113 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... [2024/05/06 19:53:31.524 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] table test.t2 not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... [2024/05/06 19:53:31.576 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:31.654 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:53:31.719 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:53:31.719 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:31.787 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] Verifying downstream PD is started... [2024/05/06 19:53:31.804 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:53:31.859 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:53:31.859 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:53:31.860 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:53:32.002 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:32.037 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... [2024/05/06 19:53:32.811 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:32.954 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] check diff successfully Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.t2 not exists for 3-th check, retry later Verifying downstream PD is started... table ddl_manager.finish_mark not exists for 48-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_gbk Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/server_config_compatibility Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:53:34.963 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:35.061 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:35.073 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:35.144 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:35.256 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] table ddl_manager.finish_mark not exists for 49-th check, retry later [2024/05/06 19:53:35.747 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.t2 not exists for 4-th check, retry later [2024/05/06 19:53:35.852 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:35.852 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:35.897 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:35.897 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:53:36.063 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:36.127 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 19:53:36.289 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:36.350 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:53:37.103 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:53:37.207 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] table ddl_manager.finish_mark not exists for 50-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t2 not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 19:53:38.301 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:53:38.318 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:38.329 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:38.365 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:38.476 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:38.482 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:38.523 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:38.545 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:38.557 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370251cc0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-rqt30, pid:1286, start at 2024-05-06 19:53:37.674112165 +0800 CST m=+5.234624148 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:37.681 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:37.651 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:37.651 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370251cc0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-rqt30, pid:1286, start at 2024-05-06 19:53:37.674112165 +0800 CST m=+5.234624148 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:37.681 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:37.651 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:37.651 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370252a0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-rqt30, pid:1380, start at 2024-05-06 19:53:37.719744231 +0800 CST m=+5.228868805 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:37.728 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:37.704 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:37.704 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:53:38.592 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:38.644 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:38.650 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:38.672 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:38.762 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37025608001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-5k7gg, pid:1220, start at 2024-05-06 19:53:37.971344355 +0800 CST m=+5.308434096 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:37.981 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:37.971 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:37.971 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 51-th check, retry later check diff failed 3-th time, retry later [2024/05/06 19:53:39.200 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:39.247 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t2 not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370269400013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-89kh8, pid:1263, start at 2024-05-06 19:53:39.180309956 +0800 CST m=+5.291065463 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:39.190 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:39.152 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:39.152 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:53:40.579 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:40.582 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:40.682 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:40.747 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:40.802 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:40.803 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:40.825 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:40.864 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:40.866 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:40.870 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:40.874 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:40.883 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:40.896 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:40.897 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:40.990 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:40.993 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37025608001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-5k7gg, pid:1220, start at 2024-05-06 19:53:37.971344355 +0800 CST m=+5.308434096 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:37.981 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:37.971 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:37.971 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370256c40002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-5k7gg, pid:1297, start at 2024-05-06 19:53:37.970100917 +0800 CST m=+5.249633789 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:37.980 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:37.969 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:37.969 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] } table ddl_manager.finish_mark not exists for 52-th check, retry later Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) check diff failed 4-th time, retry later [Mon May 6 19:53:40 CST 2024] <<<<<< START cdc server in new_ci_collation case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation.26642666.out server --log-file /tmp/tidb_cdc_test/new_ci_collation/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370279fc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-59v7t, pid:1181, start at 2024-05-06 19:53:40.248047615 +0800 CST m=+5.276571418 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:40.255 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:40.223 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:40.223 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t2 not exists for 7-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:53:41.693 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:41.765 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:41.785 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:53:42.072 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:42.073 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:42.082 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:42.087 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:42.100 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:42.172 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:42.173 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:42.184 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:42.188 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370269400013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-89kh8, pid:1263, start at 2024-05-06 19:53:39.180309956 +0800 CST m=+5.291065463 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:39.190 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:39.152 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:39.152 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37026b74000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-89kh8, pid:1327, start at 2024-05-06 19:53:39.302246615 +0800 CST m=+5.359811992 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:39.312 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:39.293 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:39.293 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/clustered_index/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/clustered_index/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:53:42.578 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:53:42.662 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:42.669 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.2611.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 19:53:42.878 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:42.878 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:42.882 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:42.891 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:42.988 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:43.000 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:43.065 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] check diff failed 5-th time, retry later [2024/05/06 19:53:43.162 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:43.183 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370279fc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-59v7t, pid:1181, start at 2024-05-06 19:53:40.248047615 +0800 CST m=+5.276571418 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:40.255 +0800 Current GC worker leader lease. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used bootstrapped True Bootstrap flag. Do not delete. tikv_gc_enable true Current GC enable status tidb_server_version 196 Bootstrap version. Do not delete. tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. system_tz Asia/Shanghai TiDB Global System Timezone. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_last_run_time 20240506-19:53:40.223 +0800 The time when last GC starts. (DO NOT EDIT) ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_safe_point 20240506-19:43:40.223 +0800 All versions after safe point can be accessed. (DO NOT EDIT) tikv_gc_leader_uuid 63d370280180014 Current GC worker leader UUID. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-dwwvr, pid:1298, start at 2024-05-06 19:53:40.657230872 +0800 CST m=+5.317457123 Host name and pid of current GC leader. (DO NOT EDIT) new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_lease 20240506-19:55:40.666 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_leader_uuid 63d370279f40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-59v7t, pid:1258, start at 2024-05-06 19:53:40.267342904 +0800 CST m=+5.240681245 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_leader_lease 20240506-19:55:40.274 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_last_run_time 20240506-19:53:40.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_safe_point 20240506-19:43:40.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. VARIABLE_NAME VARIABLE_VALUE COMMENT tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. bootstrapped True Bootstrap flag. Do not delete. tikv_gc_last_run_time 20240506-19:53:40.272 +0800 The time when last GC starts. (DO NOT EDIT) tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. tikv_gc_safe_point 20240506-19:43:40.272 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash new_collation_enabled True If the new collations are enabled. Do not edit it. Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community ddl_table_version 3 DDL Table Version. Do not delete. Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD tikv_gc_leader_uuid 63d370280180014 Current GC worker leader UUID. (DO NOT EDIT) UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-dwwvr, pid:1298, start at 2024-05-06 19:53:40.657230872 +0800 CST m=+5.317457123 Host name and pid of current GC leader. (DO NOT EDIT) Prometheus Prefix: tiflash_proxy_ Profile: release tikv_gc_leader_lease 20240506-19:55:40.666 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bank/tiflash/log/server.log tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. Logging errors to /tmp/tidb_cdc_test/bank/tiflash/log/error.log tikv_gc_last_run_time 20240506-19:53:40.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:40.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/bank/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/bank/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/bank/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t2 not exists for 8-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:53:43.485 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370281940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-dwwvr, pid:1380, start at 2024-05-06 19:53:40.74233631 +0800 CST m=+5.353125256 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:40.752 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:40.709 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:40.709 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_resign/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_resign/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 53-th check, retry later [2024/05/06 19:53:43.692 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:43.704 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:43.706 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:53:43.781 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:53:43 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2db496b2-f96d-4cb0-9359-1d2fe6d92e0c {"id":"2db496b2-f96d-4cb0-9359-1d2fe6d92e0c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996421} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc06e252d 2db496b2-f96d-4cb0-9359-1d2fe6d92e0c /tidb/cdc/default/default/upstream/7365853490070493562 {"id":7365853490070493562,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2db496b2-f96d-4cb0-9359-1d2fe6d92e0c {"id":"2db496b2-f96d-4cb0-9359-1d2fe6d92e0c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996421} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc06e252d 2db496b2-f96d-4cb0-9359-1d2fe6d92e0c /tidb/cdc/default/default/upstream/7365853490070493562 {"id":7365853490070493562,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2db496b2-f96d-4cb0-9359-1d2fe6d92e0c {"id":"2db496b2-f96d-4cb0-9359-1d2fe6d92e0c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996421} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc06e252d 2db496b2-f96d-4cb0-9359-1d2fe6d92e0c /tidb/cdc/default/default/upstream/7365853490070493562 {"id":7365853490070493562,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:53:44.089 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:53:44.096 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:53:44.176 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsdae3a44b_3917_4159_a0b8_9ee1e274d664"] [2024/05/06 19:53:44.178 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsab4470ed_cbe0_4020_a025_94bc39381dc0"] + break + set +x Create changefeed successfully! ID: de3f8c0f-162e-4214-b4f7-ec37b5b2f338 Info: {"upstream_id":7365853490070493562,"namespace":"default","id":"de3f8c0f-162e-4214-b4f7-ec37b5b2f338","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-05-06T19:53:44.134128222+08:00","start_ts":449576021734457345,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576021734457345,"checkpoint_ts":449576021734457345,"checkpoint_time":"2024-05-06 19:53:40.801"} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37028acc0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-98cvh, pid:1303, start at 2024-05-06 19:53:41.304972847 +0800 CST m=+5.316998794 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:41.315 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:41.299 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:41.299 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37028acc0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-98cvh, pid:1303, start at 2024-05-06 19:53:41.304972847 +0800 CST m=+5.316998794 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:41.315 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:41.299 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:41.299 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37028adc0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-98cvh, pid:1376, start at 2024-05-06 19:53:41.319387174 +0800 CST m=+5.273984950 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:41.327 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:41.303 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:41.303 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 19:53:44.216 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs4adef415_d1cd_40dc_93c7_8cff2b6ed51b"] [2024/05/06 19:53:44.275 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs6ed72300_86c9_409f_ad6a_d14180a86bbf"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 19:53:44 CST 2024] <<<<<< START cdc server in clustered_index case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.clustered_index.25962598.out server --log-file /tmp/tidb_cdc_test/clustered_index/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/clustered_index/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449576022395322369 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576022395322369 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [2024/05/06 19:53:44.765 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs376257c4_06fe_4b12_a5b9_81e402f22180"] [Mon May 6 19:53:44 CST 2024] <<<<<< START cdc server in changefeed_fast_fail case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.26552657.out server --log-file /tmp/tidb_cdc_test/changefeed_fast_fail/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_fast_fail/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:53:45.092 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:53:45.101 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:53:45.161 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:53:45.196 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:53:45.200 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:53:45.278 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:53:45.279 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:53:45.281 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:53:45.488 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table new_ci_collation_test.t1 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 54-th check, retry later check diff failed 6-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3702c6500012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl, pid:10586, start at 2024-05-06 19:53:45.132176146 +0800 CST m=+5.383499981 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:45.141 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:45.108 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:45.108 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3702a5900004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-54cks, pid:1311, start at 2024-05-06 19:53:43.019122599 +0800 CST m=+5.467486609 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:43.026 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:43.012 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:43.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3702a5900004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-54cks, pid:1311, start at 2024-05-06 19:53:43.019122599 +0800 CST m=+5.467486609 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:43.026 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:43.012 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:43.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3702a7280011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-54cks, pid:1380, start at 2024-05-06 19:53:43.135845663 +0800 CST m=+5.513247115 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:43.142 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:43.114 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:43.114 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 19:53:45 CST 2024] <<<<<< START cdc server in bank case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.26212623.out server --log-file /tmp/tidb_cdc_test/bank/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bank/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.t2 not exists for 9-th check, retry later [2024/05/06 19:53:45.880 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:53:45.978 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:53:45.982 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:53:45.996 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:53:46.080 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [Mon May 6 19:53:45 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.26242626.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:53:46.088 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:53:46.105 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:53:46.168 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:53:46.168 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3702c5880004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-c3sn1, pid:1286, start at 2024-05-06 19:53:45.068206768 +0800 CST m=+9.080003627 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:45.079 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:45.058 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:45.058 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3702c5880004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-c3sn1, pid:1286, start at 2024-05-06 19:53:45.068206768 +0800 CST m=+9.080003627 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:45.079 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:45.058 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:45.058 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3702bc780015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-c3sn1, pid:1369, start at 2024-05-06 19:53:44.523037181 +0800 CST m=+8.482832072 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:44.530 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:44.528 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:44.528 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/availability/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/availability/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/availability/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/availability/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/availability/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3702ad6c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-b2lwz, pid:1308, start at 2024-05-06 19:53:43.544824902 +0800 CST m=+5.401298567 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:43.552 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:43.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:43.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3702ad6c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-b2lwz, pid:1308, start at 2024-05-06 19:53:43.544824902 +0800 CST m=+5.401298567 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:43.552 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:43.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:43.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3702aee80011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-b2lwz, pid:1382, start at 2024-05-06 19:53:43.626798201 +0800 CST m=+5.422023014 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:43.633 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:43.610 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:43.610 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 19:53:46.592 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [Mon May 6 19:53:46 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.26352637.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:53:47.093 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3702baa00010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:4273, start at 2024-05-06 19:53:44.376858529 +0800 CST m=+5.369987192 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:44.384 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:44.360 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:44.360 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3702baa00010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:4273, start at 2024-05-06 19:53:44.376858529 +0800 CST m=+5.369987192 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:44.384 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:44.360 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:44.360 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3702bc140008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:4347, start at 2024-05-06 19:53:44.461041509 +0800 CST m=+5.385923050 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:44.469 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:44.453 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:44.453 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [2024/05/06 19:53:47.299 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:53:47.367 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:53:47.378 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 55-th check, retry later check diff successfully < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:53:47 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0fe49427-57ed-4cb7-8a91-ba7432cc9b81 {"id":"0fe49427-57ed-4cb7-8a91-ba7432cc9b81","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996424} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc06fe934 0fe49427-57ed-4cb7-8a91-ba7432cc9b81 /tidb/cdc/default/default/upstream/7365853495832175487 {"id":7365853495832175487,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0fe49427-57ed-4cb7-8a91-ba7432cc9b81 {"id":"0fe49427-57ed-4cb7-8a91-ba7432cc9b81","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996424} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc06fe934 0fe49427-57ed-4cb7-8a91-ba7432cc9b81 /tidb/cdc/default/default/upstream/7365853495832175487 {"id":7365853495832175487,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0fe49427-57ed-4cb7-8a91-ba7432cc9b81 {"id":"0fe49427-57ed-4cb7-8a91-ba7432cc9b81","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996424} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc06fe934 0fe49427-57ed-4cb7-8a91-ba7432cc9b81 /tidb/cdc/default/default/upstream/7365853495832175487 {"id":7365853495832175487,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: dc37422b-7b68-45b8-ab45-b937593d8add Info: {"upstream_id":7365853495832175487,"namespace":"default","id":"dc37422b-7b68-45b8-ab45-b937593d8add","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:53:47.600577927+08:00","start_ts":449576022639116289,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576022639116289,"checkpoint_ts":449576022639116289,"checkpoint_time":"2024-05-06 19:53:44.252"} [2024/05/06 19:53:47.459 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:53:47.476 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:53:47.480 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:53:47.484 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:53:47.487 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:53:47.670 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] wait process cdc.test exit for 1-th time... table test.t2 not exists for 10-th check, retry later [2024/05/06 19:53:47.881 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] table new_ci_collation_test.t1 exists table new_ci_collation_test.t2 not exists for 1-th check, retry later [2024/05/06 19:53:48.183 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:53:48.186 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:48.191 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3702c6500012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl, pid:10586, start at 2024-05-06 19:53:45.132176146 +0800 CST m=+5.383499981 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:45.141 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:45.108 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:45.108 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3702c7b80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-p5cxl, pid:10666, start at 2024-05-06 19:53:45.245686786 +0800 CST m=+5.444405457 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:45.255 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:45.248 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:45.248 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 2-th time... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2698.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:53:48 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9d1dc829-df2f-4cbc-9688-6e2ed83e78bc {"id":"9d1dc829-df2f-4cbc-9688-6e2ed83e78bc","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996425} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc070f545 9d1dc829-df2f-4cbc-9688-6e2ed83e78bc /tidb/cdc/default/default/upstream/7365853492050743554 {"id":7365853492050743554,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9d1dc829-df2f-4cbc-9688-6e2ed83e78bc {"id":"9d1dc829-df2f-4cbc-9688-6e2ed83e78bc","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996425} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc070f545 9d1dc829-df2f-4cbc-9688-6e2ed83e78bc /tidb/cdc/default/default/upstream/7365853492050743554 {"id":7365853492050743554,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9d1dc829-df2f-4cbc-9688-6e2ed83e78bc {"id":"9d1dc829-df2f-4cbc-9688-6e2ed83e78bc","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996425} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc070f545 9d1dc829-df2f-4cbc-9688-6e2ed83e78bc /tidb/cdc/default/default/upstream/7365853492050743554 {"id":7365853492050743554,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.2708.out cli changefeed create --start-ts=449576022395322369 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-fast-fail [2024/05/06 19:53:48.300 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:48.302 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:53:48.302 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:53:48.302 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:53:48.368 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.cli.2680.out cli tso query --pd=http://127.0.0.1:2379 Create changefeed successfully! ID: changefeed-fast-fail Info: {"upstream_id":7365853492050743554,"namespace":"default","id":"changefeed-fast-fail","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:53:48.449455349+08:00","start_ts":449576022395322369,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576022395322369,"checkpoint_ts":449576022395322369,"checkpoint_time":"2024-05-06 19:53:43.322"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:53:48.497 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:53:48.671 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.2623.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 19:53:48.872 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:48.881 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:48.888 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:48.901 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:53:48.904 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] wait process cdc.test exit for 3-th time... < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:53:48 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d7c573ca-ce07-4dff-a738-c0faebcba19d {"id":"d7c573ca-ce07-4dff-a738-c0faebcba19d","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996426} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc0737836 d7c573ca-ce07-4dff-a738-c0faebcba19d /tidb/cdc/default/default/upstream/7365853497945352066 {"id":7365853497945352066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d7c573ca-ce07-4dff-a738-c0faebcba19d {"id":"d7c573ca-ce07-4dff-a738-c0faebcba19d","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996426} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc0737836 d7c573ca-ce07-4dff-a738-c0faebcba19d /tidb/cdc/default/default/upstream/7365853497945352066 {"id":7365853497945352066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d7c573ca-ce07-4dff-a738-c0faebcba19d {"id":"d7c573ca-ce07-4dff-a738-c0faebcba19d","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996426} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc0737836 d7c573ca-ce07-4dff-a738-c0faebcba19d /tidb/cdc/default/default/upstream/7365853497945352066 {"id":7365853497945352066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.cli.2728.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:53:48 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6e08fa71-2a8c-4fb0-9549-59ad21622b3b {"id":"6e08fa71-2a8c-4fb0-9549-59ad21622b3b","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996425} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc072253c 6e08fa71-2a8c-4fb0-9549-59ad21622b3b /tidb/cdc/default/default/upstream/7365853499194995021 {"id":7365853499194995021,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6e08fa71-2a8c-4fb0-9549-59ad21622b3b {"id":"6e08fa71-2a8c-4fb0-9549-59ad21622b3b","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996425} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc072253c 6e08fa71-2a8c-4fb0-9549-59ad21622b3b /tidb/cdc/default/default/upstream/7365853499194995021 {"id":7365853499194995021,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6e08fa71-2a8c-4fb0-9549-59ad21622b3b {"id":"6e08fa71-2a8c-4fb0-9549-59ad21622b3b","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996425} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc072253c 6e08fa71-2a8c-4fb0-9549-59ad21622b3b /tidb/cdc/default/default/upstream/7365853499194995021 {"id":7365853499194995021,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.cli.2682.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ Create changefeed successfully! ID: 895b1d99-21e8-4b08-ab01-7ba197d7774c Info: {"upstream_id":7365853499194995021,"namespace":"default","id":"895b1d99-21e8-4b08-ab01-7ba197d7774c","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-06T19:53:49.012355383+08:00","start_ts":449576023863590914,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576023863590914,"checkpoint_ts":449576023863590914,"checkpoint_time":"2024-05-06 19:53:48.923"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:53:48.970 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:49.059 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:49.059 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:53:49.185 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table ddl_manager.finish_mark not exists for 56-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:53:49.380 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:53:49 CST 2024] <<<<<< run test case changefeed_pause_resume success! >>>>>> Create changefeed successfully! ID: 9a1e0848-1913-43bd-b0f4-f0bba529926b Info: {"upstream_id":7365853497945352066,"namespace":"default","id":"9a1e0848-1913-43bd-b0f4-f0bba529926b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:53:49.333669817+08:00","start_ts":449576023940136964,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576023940136964,"checkpoint_ts":449576023940136964,"checkpoint_time":"2024-05-06 19:53:49.215"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:53:49 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/88b1b9b7-226c-45c0-8c8b-2d40980eb74b {"id":"88b1b9b7-226c-45c0-8c8b-2d40980eb74b","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996426} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc079de35 88b1b9b7-226c-45c0-8c8b-2d40980eb74b /tidb/cdc/default/default/upstream/7365853500178616945 {"id":7365853500178616945,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/88b1b9b7-226c-45c0-8c8b-2d40980eb74b {"id":"88b1b9b7-226c-45c0-8c8b-2d40980eb74b","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996426} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc079de35 88b1b9b7-226c-45c0-8c8b-2d40980eb74b /tidb/cdc/default/default/upstream/7365853500178616945 {"id":7365853500178616945,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/88b1b9b7-226c-45c0-8c8b-2d40980eb74b {"id":"88b1b9b7-226c-45c0-8c8b-2d40980eb74b","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996426} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc079de35 88b1b9b7-226c-45c0-8c8b-2d40980eb74b /tidb/cdc/default/default/upstream/7365853500178616945 {"id":7365853500178616945,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/06 19:53:49.586 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:49.587 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:49.588 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:49.591 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:49.594 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:53:49.668 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:49.701 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:53:49.705 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] + set +x + tso='449576023703420929 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576023703420929 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:53:49 CST 2024] <<<<<< START cdc server in many_pk_or_uk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.27402742.out server --log-file /tmp/tidb_cdc_test/many_pk_or_uk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/many_pk_or_uk/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' table test.t2 not exists for 11-th check, retry later [Mon May 6 19:53:49 CST 2024] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.57125714.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table new_ci_collation_test.t2 exists table new_ci_collation_test.t3 not exists for 1-th check, retry later + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-fast-fail failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-fast-fail + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-fast-fail -s + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.cli.11914.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 19:53:50.268 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] + info='{ "upstream_id": 7365853492050743554, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 449576022395322369, "checkpoint_time": "2024-05-06 19:53:43.322", "error": { "time": "2024-05-06T19:53:48.462844057+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7365853492050743554, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 449576022395322369, "checkpoint_time": "2024-05-06 19:53:43.322", "error": { "time": "2024-05-06T19:53:48.462844057+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7365853492050743554, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 449576022395322369, "checkpoint_time": "2024-05-06 19:53:43.322", "error": { "time": "2024-05-06T19:53:48.462844057+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365853492050743554, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 449576022395322369, '"checkpoint_time":' '"2024-05-06' '19:53:43.322",' '"error":' '{' '"time":' '"2024-05-06T19:53:48.462844057+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7365853492050743554, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 449576022395322369, '"checkpoint_time":' '"2024-05-06' '19:53:43.322",' '"error":' '{' '"time":' '"2024-05-06T19:53:48.462844057+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully + set +x + tso='449576023822172161 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576023822172161 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:53:50 CST 2024] <<<<<< START cdc server in tidb_mysql_test case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.27132715.out server --log-file /tmp/tidb_cdc_test/tidb_mysql_test/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tidb_mysql_test/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:53:50.393 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] + set +x + GO111MODULE=on + go run bank.go case.go -u 'root@tcp(127.0.0.1:4000)/bank' -d 'root@tcp(127.0.0.1:3306)/bank' --test-round=20000 -a 127.0.0.1:20080 go: downloading golang.org/x/sync v0.7.0 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading google.golang.org/grpc v1.62.1 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 [2024/05/06 19:53:50.690 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:50.691 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:50.694 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:50.699 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:50.774 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:50.783 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:50.785 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:53:50.787 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] + set +x + tso='449576023807492098 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576023807492098 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x run test case test_kill_owner [Mon May 6 19:53:50 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.26772679.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.uber.org/multierr v1.11.0 + set +x go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda [Pipeline] } [2024/05/06 19:53:51.073 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:53:51.285 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:53:51.477 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:51.565 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:51.566 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:53:51.569 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:51.615 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] wait process cdc.test exit for 1-th time... table test.t2 exists check diff successfully table ddl_manager.finish_mark not exists for 57-th check, retry later table new_ci_collation_test.t3 exists table new_ci_collation_test.t4 not exists for 1-th check, retry later [2024/05/06 19:53:51.691 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:53:51.812 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] wait process cdc.test exit for 2-th time... Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) [2024/05/06 19:53:51.923 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:51.923 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:51.926 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:53:52.102 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] + set +x + tso='449576024252874753 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576024252874753 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:53:52 CST 2024] <<<<<< START cdc server in server_config_compatibility case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.1196611968.out server --log-file /tmp/tidb_cdc_test/server_config_compatibility/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/server_config_compatibility/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/server_config_compatibility/conf/server.toml ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table owner_resign.t1 exists check diff failed 1-th time, retry later [2024/05/06 19:53:52.202 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:53:52.388 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:53:52.396 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:53:52 CST 2024] <<<<<< run test case changefeed_fast_fail success! >>>>>> [2024/05/06 19:53:52.597 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] wait process cdc.test exit for 2-th time... check diff failed 1-th time, retry later table clustered_index_test.t0 exists table clustered_index_test.t1 not exists for 1-th check, retry later [2024/05/06 19:53:52.704 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:53:52.871 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:53:52.875 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:53:52 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a93da9b-b745-4a01-a550-376337f9b95a {"id":"4a93da9b-b745-4a01-a550-376337f9b95a","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996429} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc085cf32 4a93da9b-b745-4a01-a550-376337f9b95a /tidb/cdc/default/default/upstream/7365853516468347035 {"id":7365853516468347035,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a93da9b-b745-4a01-a550-376337f9b95a {"id":"4a93da9b-b745-4a01-a550-376337f9b95a","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996429} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc085cf32 4a93da9b-b745-4a01-a550-376337f9b95a /tidb/cdc/default/default/upstream/7365853516468347035 {"id":7365853516468347035,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a93da9b-b745-4a01-a550-376337f9b95a {"id":"4a93da9b-b745-4a01-a550-376337f9b95a","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996429} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc085cf32 4a93da9b-b745-4a01-a550-376337f9b95a /tidb/cdc/default/default/upstream/7365853516468347035 {"id":7365853516468347035,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/06 19:53:52.955 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:53:53.035 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:53:52 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb3d3de2-e310-40a0-bee3-85129eced2a5 {"id":"eb3d3de2-e310-40a0-bee3-85129eced2a5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996430} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc07c403b eb3d3de2-e310-40a0-bee3-85129eced2a5 /tidb/cdc/default/default/upstream/7365853514118022637 {"id":7365853514118022637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb3d3de2-e310-40a0-bee3-85129eced2a5 {"id":"eb3d3de2-e310-40a0-bee3-85129eced2a5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996430} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc07c403b eb3d3de2-e310-40a0-bee3-85129eced2a5 /tidb/cdc/default/default/upstream/7365853514118022637 {"id":7365853514118022637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb3d3de2-e310-40a0-bee3-85129eced2a5 {"id":"eb3d3de2-e310-40a0-bee3-85129eced2a5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996430} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc07c403b eb3d3de2-e310-40a0-bee3-85129eced2a5 /tidb/cdc/default/default/upstream/7365853514118022637 {"id":7365853514118022637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2795.out cli changefeed create --start-ts=449576023703420929 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370326240002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-fd5nv, pid:1194, start at 2024-05-06 19:53:51.248488223 +0800 CST m=+14.423313300 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:51.346 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:51.396 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:51.396 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370326240002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-fd5nv, pid:1194, start at 2024-05-06 19:53:51.248488223 +0800 CST m=+14.423313300 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:51.346 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:51.396 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:51.396 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37032ab80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-fd5nv, pid:1221, start at 2024-05-06 19:53:51.57766878 +0800 CST m=+14.558715011 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:55:51.669 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:53:51.683 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:43:51.683 +0800 All versions after safe point can be accessed. (DO NOT EDIT) wait process cdc.test exit for 3-th time... [2024/05/06 19:53:53.251 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:53:53.350 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] Create changefeed successfully! ID: 8f4a83dc-d548-4988-b2c5-1dc4d0b84eb7 Info: {"upstream_id":7365853514118022637,"namespace":"default","id":"8f4a83dc-d548-4988-b2c5-1dc4d0b84eb7","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:53:53.377310468+08:00","start_ts":449576023703420929,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576023703420929,"checkpoint_ts":449576023703420929,"checkpoint_time":"2024-05-06 19:53:48.312"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:53:53 CST 2024] <<<<<< run test case resolve_lock success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:53:53 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4ade809-94ec-45fb-9277-0e74a2c16712 {"id":"a4ade809-94ec-45fb-9277-0e74a2c16712","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996430} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc082523c a4ade809-94ec-45fb-9277-0e74a2c16712 /tidb/cdc/default/default/upstream/7365853507938490129 {"id":7365853507938490129,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4ade809-94ec-45fb-9277-0e74a2c16712 {"id":"a4ade809-94ec-45fb-9277-0e74a2c16712","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996430} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc082523c a4ade809-94ec-45fb-9277-0e74a2c16712 /tidb/cdc/default/default/upstream/7365853507938490129 {"id":7365853507938490129,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4ade809-94ec-45fb-9277-0e74a2c16712 {"id":"a4ade809-94ec-45fb-9277-0e74a2c16712","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996430} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc082523c a4ade809-94ec-45fb-9277-0e74a2c16712 /tidb/cdc/default/default/upstream/7365853507938490129 {"id":7365853507938490129,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 142 0 0 100 142 0 1470 --:--:-- --:--:-- --:--:-- 1479 mysql_test start go: downloading github.com/maxshuang/mysql-tester v0.0.2 [2024/05/06 19:53:53.552 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:53:53.653 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] table new_ci_collation_test.t4 exists table new_ci_collation_test.t5 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 58-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:53:53 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fb919b0e-f307-46df-a7ec-c9d39bdecdb0 {"id":"fb919b0e-f307-46df-a7ec-c9d39bdecdb0","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996431} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e45f fb919b0e-f307-46df-a7ec-c9d39bdecdb0 /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fb919b0e-f307-46df-a7ec-c9d39bdecdb0 {"id":"fb919b0e-f307-46df-a7ec-c9d39bdecdb0","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996431} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e45f fb919b0e-f307-46df-a7ec-c9d39bdecdb0 /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fb919b0e-f307-46df-a7ec-c9d39bdecdb0 {"id":"fb919b0e-f307-46df-a7ec-c9d39bdecdb0","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996431} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e45f fb919b0e-f307-46df-a7ec-c9d39bdecdb0 /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.2732.out cli changefeed create --start-ts=449576023807492098 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table clustered_index_test.t1 exists table clustered_index_test.t2 not exists for 1-th check, retry later [2024/05/06 19:53:53.951 +08:00] [INFO] [main.go:812] ["testMultiDDLs take %v1m17.250886343s"] [2024/05/06 19:53:54.035 +08:00] [INFO] [main.go:74] ["DefaultValue integration tests take 1m17.335726859s"] table consistent_replicate_gbk.GBKTABLE not exists for 1-th check, retry later table mark.finish_mark_1 not exists for 1-th check, retry later Create changefeed successfully! ID: 15d744e6-9542-4a9e-820f-73b6ebaa1984 Info: {"upstream_id":7365853495484688586,"namespace":"default","id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:53:54.37838987+08:00","start_ts":449576023807492098,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576023807492098,"checkpoint_ts":449576023807492098,"checkpoint_time":"2024-05-06 19:53:48.709"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... check diff successfully check diff successfully [Mon May 6 19:53:54 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerDontUpdateCheckpoint=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.29192921.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:53:55 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97ac4a8f-2e95-4b8c-8cdb-d23097209b58 {"id":"97ac4a8f-2e95-4b8c-8cdb-d23097209b58","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996432} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc088bc4e 97ac4a8f-2e95-4b8c-8cdb-d23097209b58 /tidb/cdc/default/default/upstream/7365853520687345374 {"id":7365853520687345374,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97ac4a8f-2e95-4b8c-8cdb-d23097209b58 {"id":"97ac4a8f-2e95-4b8c-8cdb-d23097209b58","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996432} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc088bc4e 97ac4a8f-2e95-4b8c-8cdb-d23097209b58 /tidb/cdc/default/default/upstream/7365853520687345374 {"id":7365853520687345374,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97ac4a8f-2e95-4b8c-8cdb-d23097209b58 {"id":"97ac4a8f-2e95-4b8c-8cdb-d23097209b58","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996432} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc088bc4e 97ac4a8f-2e95-4b8c-8cdb-d23097209b58 /tidb/cdc/default/default/upstream/7365853520687345374 {"id":7365853520687345374,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.cli.12027.out cli changefeed create --start-ts=449576024252874753 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ table ddl_manager.finish_mark not exists for 59-th check, retry later table new_ci_collation_test.t5 exists check diff failed 1-th time, retry later go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda Create changefeed successfully! ID: 003cc218-6250-4bc4-be47-a4533b4fcb67 Info: {"upstream_id":7365853520687345374,"namespace":"default","id":"003cc218-6250-4bc4-be47-a4533b4fcb67","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:53:55.635778735+08:00","start_ts":449576024252874753,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576024252874753,"checkpoint_ts":449576024252874753,"checkpoint_time":"2024-05-06 19:53:50.408"} PASS go: downloading golang.org/x/text v0.14.0 coverage: 2.6% of statements in github.com/pingcap/tiflow/... + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2461.out cli tso query --pd=http://127.0.0.1:2379 table clustered_index_test.t2 exists check table exists success check diff failed 1-th time, retry later go: downloading github.com/pingcap/tidb/parser v0.0.0-20211122065349-4d16aabf900b go: downloading github.com/pingcap/errors v0.11.5-0.20211009033009-93128226aaa3 go: downloading github.com/sirupsen/logrus v1.8.1 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading go.uber.org/atomic v1.9.0 go: downloading golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 go: downloading golang.org/x/text v0.3.7 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading go.uber.org/zap v1.19.1 go: downloading github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 go: downloading go.uber.org/multierr v1.7.0 table consistent_replicate_gbk.GBKTABLE exists table consistent_replicate_gbk.check1 not exists for 1-th check, retry later table mark.finish_mark_1 not exists for 2-th check, retry later + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:53:57 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/9a1e0848-1913-43bd-b0f4-f0bba529926b {UpstreamID:7365853497945352066 Namespace:default ID:9a1e0848-1913-43bd-b0f4-f0bba529926b SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:49.333669817 +0800 CST StartTs:449576023940136964 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003cda3f0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576023966089219} {CheckpointTs:449576025814204429 MinTableBarrierTs:449576025814204429 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2322f7d7-f7c8-4485-b673-f6582f23df94 {"id":"2322f7d7-f7c8-4485-b673-f6582f23df94","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996434} /tidb/cdc/default/__cdc_meta__/capture/d7c573ca-ce07-4dff-a738-c0faebcba19d {"id":"d7c573ca-ce07-4dff-a738-c0faebcba19d","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996426} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc0737836 d7c573ca-ce07-4dff-a738-c0faebcba19d /tidb/cdc/default/__cdc_meta__/owner/22318f4dc07378c5 2322f7d7-f7c8-4485-b673-f6582f23df94 /tidb/cdc/default/default/changefeed/info/9a1e0848-1913-43bd-b0f4-f0bba529926b {"upstream-id":7365853497945352066,"namespace":"default","changefeed-id":"9a1e0848-1913-43bd-b0f4-f0bba529926b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:49.333669817+08:00","start-ts":449576023940136964,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576023966089219} /tidb/cdc/default/default/changefeed/status/9a1e0848-1913-43bd-b0f4-f0bba529926b {"checkpoint-ts":449576025814204429,"min-table-barrier-ts":449576025814204429,"admin-job-type":0} /tidb/cdc/default/default/task/position/2322f7d7-f7c8-4485-b673-f6582f23df94/9a1e0848-1913-43bd-b0f4-f0bba529926b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d7c573ca-ce07-4dff-a738-c0faebcba19d/9a1e0848-1913-43bd-b0f4-f0bba529926b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853497945352066 {"id":7365853497945352066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/9a1e0848-1913-43bd-b0f4-f0bba529926b {UpstreamID:7365853497945352066 Namespace:default ID:9a1e0848-1913-43bd-b0f4-f0bba529926b SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:49.333669817 +0800 CST StartTs:449576023940136964 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003cda3f0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576023966089219} {CheckpointTs:449576025814204429 MinTableBarrierTs:449576025814204429 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2322f7d7-f7c8-4485-b673-f6582f23df94 {"id":"2322f7d7-f7c8-4485-b673-f6582f23df94","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996434} /tidb/cdc/default/__cdc_meta__/capture/d7c573ca-ce07-4dff-a738-c0faebcba19d {"id":"d7c573ca-ce07-4dff-a738-c0faebcba19d","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996426} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc0737836 d7c573ca-ce07-4dff-a738-c0faebcba19d /tidb/cdc/default/__cdc_meta__/owner/22318f4dc07378c5 2322f7d7-f7c8-4485-b673-f6582f23df94 /tidb/cdc/default/default/changefeed/info/9a1e0848-1913-43bd-b0f4-f0bba529926b {"upstream-id":7365853497945352066,"namespace":"default","changefeed-id":"9a1e0848-1913-43bd-b0f4-f0bba529926b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:49.333669817+08:00","start-ts":449576023940136964,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576023966089219} /tidb/cdc/default/default/changefeed/status/9a1e0848-1913-43bd-b0f4-f0bba529926b {"checkpoint-ts":449576025814204429,"min-table-barrier-ts":449576025814204429,"admin-job-type":0} /tidb/cdc/default/default/task/position/2322f7d7-f7c8-4485-b673-f6582f23df94/9a1e0848-1913-43bd-b0f4-f0bba529926b {"checkpoint-ts":0,"resolved-ts":0,"co+ grep -q 'failed to get info:' unt":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d7c573ca-ce07-4dff-a738-c0faebcba19d/9a1e0848-1913-43bd-b0f4-f0bba529926b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853497945352066 {"id":7365853497945352066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/9a1e0848-1913-43bd-b0f4-f0bba529926b {UpstreamID:7365853497945352066 Namespace:default ID:9a1e0848-1913-43bd-b0f4-f0bba529926b SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:49.333669817 +0800 CST StartTs:449576023940136964 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003cda3f0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576023966089219} {CheckpointTs:449576025814204429 MinTableBarrierTs:449576025814204429 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2322f7d7-f7c8-4485-b673-f6582f23df94 {"id":"2322f7d7-f7c8-4485-b673-f6582f23df94","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996434} /tidb/cdc/default/__cdc_meta__/capture/d7c573ca-ce07-4dff-a738-c0faebcba19d {"id":"d7c573ca-ce07-4dff-a738-c0faebcba19d","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996426} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc0737836 d7c573ca-ce07-4dff-a738-c0faebcba19d /tidb/cdc/default/__cdc_meta__/owner/22318f4dc07378c5 2322f7d7-f7c8-4485-b673-f6582f23df94 /tidb/cdc/default/default/changefeed/info/9a1e0848-1913-43bd-b0f4-f0bba529926b {"upstream-id":7365853497945352066,"namespace":"default","changefeed-id":"9a1e0848-1913-43bd-b0f4-f0bba529926b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:49.333669817+08:00","start-ts":449576023940136964,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576023966089219} /tidb/cdc/default/default/changefeed/status/9a1e0848-1913-43bd-b0f4-f0bba529926b {"checkpoint-ts":449576025814204429,"min-table-barrier-ts":449576025814204429,"admin-job-type":0} /tidb/cdc/default/default/task/position/2322f7d7-f7c8-4485-b673-f6582f23df94/9a1e0848-1913-43bd-b0f4-f0bba529926b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d7c573ca-ce07-4dff-a738-c0faebcba19d/9a1e0848-1913-43bd-b0f4-f0bba529926b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853497945352066 {"id":7365853497945352066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^2$' "is-owner": true, run task successfully table ddl_manager.finish_mark not exists for 60-th check, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later 2 run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading golang.org/x/sync v0.7.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/stretchr/testify v1.9.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading google.golang.org/api v0.170.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 check diff failed 2-th time, retry later go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 table mark.finish_mark_1 not exists for 3-th check, retry later go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible check diff failed 2-th time, retry later + set +x + tso='449576025873711106 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576025873711106 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:53:58 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.24822484.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-logr/stdr v1.2.2 table consistent_replicate_gbk.check1 exists check diff successfully wait process cdc.test exit for 1-th time... owner pid: 2682 owner id fb919b0e-f307-46df-a7ec-c9d39bdecdb0 [Mon May 6 19:53:58 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.28432845.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/cli_with_auth/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] // cache wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... table ddl_manager.finish_mark not exists for 61-th check, retry later check diff failed 3-th time, retry later check diff failed 3-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:54:00 CST 2024] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.59285930.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_1 not exists for 4-th check, retry later check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 62-th check, retry later check data successfully wait process cdc.test exit for 1-th time... check diff failed 4-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:01 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/786a0fe3-9a1f-4d79-8fbd-d794ec276f85 {"id":"786a0fe3-9a1f-4d79-8fbd-d794ec276f85","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996438} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc0802556 786a0fe3-9a1f-4d79-8fbd-d794ec276f85 /tidb/cdc/default/default/upstream/7365853506087405840 {"id":7365853506087405840,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/786a0fe3-9a1f-4d79-8fbd-d794ec276f85 {"id":"786a0fe3-9a1f-4d79-8fbd-d794ec276f85","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996438} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc0802556 786a0fe3-9a1f-4d79-8fbd-d794ec276f85 /tidb/cdc/default/default/upstream/7365853506087405840 {"id":7365853506087405840,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/786a0fe3-9a1f-4d79-8fbd-d794ec276f85 {"id":"786a0fe3-9a1f-4d79-8fbd-d794ec276f85","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996438} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc0802556 786a0fe3-9a1f-4d79-8fbd-d794ec276f85 /tidb/cdc/default/default/upstream/7365853506087405840 {"id":7365853506087405840,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 2031be2b-6b5f-457a-82d2-50b232272e1c Info: {"upstream_id":7365853506087405840,"namespace":"default","id":"2031be2b-6b5f-457a-82d2-50b232272e1c","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:54:01.726129564+08:00","start_ts":449576025873711106,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576025873711106,"checkpoint_ts":449576025873711106,"checkpoint_time":"2024-05-06 19:53:56.591"} check diff failed 4-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/batch_update_to_no_batch/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:01 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0024d30e0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576024319459331 MinTableBarrierTs:449576027071184904 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576024319459331, checkpointTs: 449576024319459331, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b8b9f577-d65c-407e-b33b-5a931e549c0e {"id":"b8b9f577-d65c-407e-b33b-5a931e549c0e","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996439} /tidb/cdc/default/__cdc_meta__/capture/fb919b0e-f307-46df-a7ec-c9d39bdecdb0 {"id":"fb919b0e-f307-46df-a7ec-c9d39bdecdb0","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996431} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e45f fb919b0e-f307-46df-a7ec-c9d39bdecdb0 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e4c9 b8b9f577-d65c-407e-b33b-5a931e549c0e /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576024319459331,"min-table-barrier-ts":449576027071184904,"admin-job-type":0} /tidb/cdc/default/default/task/position/b8b9f577-d65c-407e-b33b-5a931e549c0e/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/fb919b0e-f307-46df-a7ec-c9d39bdecdb0/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0024d30e0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576024319459331 MinTableBarrierTs:449576027071184904 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576024319459331, checkpointTs: 449576024319459331, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b8b9f577-d65c-407e-b33b-5a931e549c0e {"id":"b8b9f577-d65c-407e-b33b-5a931e549c0e","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996439} /tidb/cdc/default/__cdc_meta__/capture/fb919b0e-f307-46df-a7ec-c9d39bdecdb0 {"id":"fb919b0e-f307-46df-a7ec-c9d39bdecdb0","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996431} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e45f fb919b0e-f307-46df-a7ec-c9d39bdecdb0 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e4c9 b8b9f577-d65c-407e-b33b-5a931e549c0e /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576024319459331,"min-table-barrier-ts":449576027071184904,"admin-job-type":0} /tidb/cdc/default/default/task/position/b8b9f577-d65c-407e-b33b-5a931e549c0e/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'failed to get info:' /tidb/cdc/default/default/task/position/fb919b0e-f307-46df-a7ec-c9d39bdecdb0/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0024d30e0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576024319459331 MinTableBarrierTs:449576027071184904 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576024319459331, checkpointTs: 449576024319459331, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b8b9f577-d65c-407e-b33b-5a931e549c0e {"id":"b8b9f577-d65c-407e-b33b-5a931e549c0e","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996439} /tidb/cdc/default/__cdc_meta__/capture/fb919b0e-f307-46df-a7ec-c9d39bdecdb0 {"id":"fb919b0e-f307-46df-a7ec-c9d39bdecdb0","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996431} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e45f fb919b0e-f307-46df-a7ec-c9d39bdecdb0 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e4c9 b8b9f577-d65c-407e-b33b-5a931e549c0e /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576024319459331,"min-table-barrier-ts":449576027071184904,"admin-job-type":0} /tidb/cdc/default/default/task/position/b8b9f577-d65c-407e-b33b-5a931e549c0e/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'etcd info' /tidb/cdc/default/default/task/position/fb919b0e-f307-46df-a7ec-c9d39bdecdb0/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "fb919b0e-f307-46df-a7ec-c9d39bdecdb0" | grep -v cluster_id | grep id go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 start tidb cluster in /tmp/tidb_cdc_test/cli_with_auth Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... table mark.finish_mark_1 exists table mark.finish_mark_2 not exists for 1-th check, retry later wait process cdc.test exit for 3-th time... [2024/05/06 19:54:01.369 +08:00] [INFO] [case.go:445] ["open db success"] [dsn=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:01.370 +08:00] [INFO] [case.go:445] ["open db success"] [dsn=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:54:01.370 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq0] [2024/05/06 19:54:01.486 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.759145ms"] [query="INSERT IGNORE INTO accounts_seq0 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/06 19:54:01.486 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq1] [2024/05/06 19:54:01.571 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.487326ms"] [query="INSERT IGNORE INTO accounts_seq1 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/06 19:54:01.571 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq2] [2024/05/06 19:54:01.654 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.400994ms"] [query="INSERT IGNORE INTO accounts_seq2 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/06 19:54:01.654 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq3] [2024/05/06 19:54:01.738 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.701737ms"] [query="INSERT IGNORE INTO accounts_seq3 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/06 19:54:01.738 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq4] [2024/05/06 19:54:01.822 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.727059ms"] [query="INSERT IGNORE INTO accounts_seq4 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/06 19:54:01.822 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts0] [2024/05/06 19:54:01.907 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.76669ms"] [query="INSERT IGNORE INTO accounts0 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/06 19:54:01.908 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts1] [2024/05/06 19:54:01.989 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.05787ms"] [query="INSERT IGNORE INTO accounts1 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/06 19:54:01.990 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts2] [2024/05/06 19:54:02.073 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.017744ms"] [query="INSERT IGNORE INTO accounts2 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/06 19:54:02.073 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts3] [2024/05/06 19:54:02.157 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.510248ms"] [query="INSERT IGNORE INTO accounts3 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/06 19:54:02.157 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts4] [2024/05/06 19:54:02.243 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.702589ms"] [query="INSERT IGNORE INTO accounts4 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [Pipeline] } "cluster-id": "default" "id": "b8b9f577-d65c-407e-b33b-5a931e549c0e", "cluster-id": "default" run task successfully check diff failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:54:03 CST 2024] <<<<<< run test case server_config_compatibility success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:03 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e6ed71e4-9ccf-46e5-8cb8-d9a7df125054 {UpstreamID:7365853516468347035 Namespace:default ID:e6ed71e4-9ccf-46e5-8cb8-d9a7df125054 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:52.624436867 +0800 CST StartTs:449576024804163587 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003119d40 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576024830115843} {CheckpointTs:449576025131581445 MinTableBarrierTs:449576027569782793 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576025131581445, checkpointTs: 449576025131581445, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576025131581445, checkpointTs: 449576025131581445, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc973080-7dbe-40f3-9aad-c84f15a7987f {"id":"cc973080-7dbe-40f3-9aad-c84f15a7987f","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996440} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc085cfdd cc973080-7dbe-40f3-9aad-c84f15a7987f /tidb/cdc/default/default/changefeed/info/e6ed71e4-9ccf-46e5-8cb8-d9a7df125054 {"upstream-id":7365853516468347035,"namespace":"default","changefeed-id":"e6ed71e4-9ccf-46e5-8cb8-d9a7df125054","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:52.624436867+08:00","start-ts":449576024804163587,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576024830115843} /tidb/cdc/default/default/changefeed/status/e6ed71e4-9ccf-46e5-8cb8-d9a7df125054 {"checkpoint-ts":449576025131581445,"min-table-barrier-ts":449576027569782793,"admin-job-type":0} /tidb/cdc/default/default/task/position/cc973080-7dbe-40f3-9aad-c84f15a7987f/e6ed71e4-9ccf-46e5-8cb8-d9a7df125054 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853516468347035 {"id":7365853516468347035,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e6ed71e4-9ccf-46e5-8cb8-d9a7df125054 {UpstreamID:7365853516468347035 Namespace:default ID:e6ed71e4-9ccf-46e5-8cb8-d9a7df125054 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:52.624436867 +0800 CST StartTs:449576024804163587 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003119d40 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576024830115843} {CheckpointTs:449576025131581445 MinTableBarrierTs:449576027569782793 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576025131581445, checkpointTs: 449576025131581445, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576025131581445, checkpointTs: 449576025131581445, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc973080-7dbe-40f3-9aad-c84f15a7987f {"id":"cc973080-7dbe-40f3-9aad-c84f15a7987f","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996440} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc085cfdd cc973080-7dbe-40f3-9aad-c84f15a7987f /tidb/cdc/default/default/changefeed/info/e6ed71e4-9ccf-46e5-8cb8-d9a7df125054 {"upstream-id":7365853516468347035,"namespace":"default","changefeed-id":"e6ed71e4-9ccf-46e5-8cb8-d9a7df125054","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:52.624436867+08:00","start-ts":449576024804163587,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576024830115843} /tidb/cdc/default/default/changefeed/status/e6ed71e4-9ccf-46e5-8cb8-d9a7df125054 {"checkpoint-ts":449576025131581445,"min-table-barrier-ts":449576027569782793,"admin-job-type":0} /tidb/cdc/default/default/task/position/cc973080-7dbe-40f3-9aad-c84f15a7987f/e6ed71e4-9ccf-46e5-8cb8-d9a7df125054 {"checkpoint-ts":0,"resolved-ts":0,"count+ grep -q 'failed to get info:' ":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853516468347035 {"id":7365853516468347035,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e6ed71e4-9ccf-46e5-8cb8-d9a7df125054 {UpstreamID:7365853516468347035 Namespace:default ID:e6ed71e4-9ccf-46e5-8cb8-d9a7df125054 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:52.624436867 +0800 CST StartTs:449576024804163587 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003119d40 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576024830115843} {CheckpointTs:449576025131581445 MinTableBarrierTs:449576027569782793 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576025131581445, checkpointTs: 449576025131581445, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576025131581445, checkpointTs: 449576025131581445, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc973080-7dbe-40f3-9aad-c84f15a7987f {"id":"cc973080-7dbe-40f3-9aad-c84f15a7987f","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996440} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc085cfdd cc973080-7dbe-40f3-9aad-c84f15a7987f /tidb/cdc/default/default/changefeed/info/e6ed71e4-9ccf-46e5-8cb8-d9a7df125054 {"upstream-id":7365853516468347035,"namespace":"default","changefeed-id":"e6ed71e4-9ccf-46e5-8cb8-d9a7df125054","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:52.624436867+08:00","start-ts":449576024804163587,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576024830115843} /tidb/cdc/default/default/changefeed/status/e6ed71e4-9ccf-46e5-8cb8-d9a7df125054 {"checkpoint-ts":449576025131581445,"min-table-barrier-ts":449576027569782793,"admin-job-type":0} /tidb/cdc/default/default/task/position/cc973080-7dbe-40f3-9aad-c84f15a7987f/e6ed71e4-9ccf-46e5-8cb8-d9a7df125054 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853516468347035 {"id":7365853516468347035,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table ddl_manager.finish_mark not exists for 63-th check, retry later table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd [Pipeline] // cache check diff failed 5-th time, retry later [2024/05/06 19:54:04.325 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] table mark.finish_mark_2 not exists for 2-th check, retry later check diff failed 5-th time, retry later capture_id: b8b9f577-d65c-407e-b33b-5a931e549c0e wait process 2682 exit for 1-th time... check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 table ddl_manager.finish_mark not exists for 64-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/batch_update_to_no_batch Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table cdc_tiflash_test.multi_data_type exists check diff successfully wait process 2682 exit for 2-th time... check diff successfully check diff failed 1-th time, retry later wait process cdc.test exit for 1-th time... [2024/05/06 19:54:06.337 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] wait process 2682 exit for 3-th time... table mark.finish_mark_2 not exists for 3-th check, retry later check diff failed 6-th time, retry later [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (2682) - No such process wait process 2682 exit for 4-th time... process 2682 already exit cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 |grep b8b9f577-d65c-407e-b33b-5a931e549c0e -A1 | grep '"is-owner": true' table ddl_manager.finish_mark not exists for 65-th check, retry later wait process cdc.test exit for 3-th time... start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... [Pipeline] // dir cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Pipeline] } [Mon May 6 19:54:07 CST 2024] <<<<<< run test case tiflash success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 19:54:08.348 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] "is-owner": true, run task successfully test_kill_owner: pass table mark.finish_mark_2 not exists for 4-th check, retry later check diff failed 7-th time, retry later [Pipeline] // dir check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 1-th time... [Pipeline] } [Pipeline] // withCredentials [Pipeline] } Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... [Pipeline] // withCredentials [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // timeout [Pipeline] } [Pipeline] // timeout [Pipeline] } wait process cdc.test exit for 3-th time... [Pipeline] // stage [Pipeline] } wait process cdc.test exit for 3-th time... table ddl_manager.finish_mark not exists for 66-th check, retry later [Pipeline] // stage [Pipeline] } Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // container [Pipeline] } check_redo_resolved_ts e6ed71e4-9ccf-46e5-8cb8-d9a7df125054 449576029168599046 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/e6ed71e4-9ccf-46e5-8cb8-d9a7df125054/meta global resolved ts 449576027700592666 not forward to 449576029168599046 run task failed 1-th time, retry later [Pipeline] // container [Pipeline] } cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit run test case test_hang_up_owner [Mon May 6 19:54:10 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.30703072.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // withEnv [Pipeline] } cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:54:10 CST 2024] <<<<<< run test case new_ci_collation success! >>>>>> [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // node [2024/05/06 19:54:10.360 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] [Pipeline] } [Pipeline] // podTemplate table mark.finish_mark_2 not exists for 5-th check, retry later [Pipeline] } check diff failed 8-th time, retry later [Pipeline] // podTemplate [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // withEnv [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check_redo_resolved_ts e6ed71e4-9ccf-46e5-8cb8-d9a7df125054 449576029168599046 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/e6ed71e4-9ccf-46e5-8cb8-d9a7df125054/meta run task successfully [Pipeline] // stage [Pipeline] } table ddl_manager.finish_mark not exists for 67-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... Starting Upstream TiDB... [2024/05/06 19:54:12.373 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] table mark.finish_mark_2 not exists for 6-th check, retry later check diff failed 9-th time, retry later Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:54:13 CST 2024] <<<<<< run test case kafka_big_messages success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:13 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b19dd0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576027857616905 MinTableBarrierTs:449576030216912903 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576027857616905, checkpointTs: 449576027857616905, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576027857616905, checkpointTs: 449576027857616905, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576027857616905, checkpointTs: 449576027857616905, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bfc2cf17-0ecb-47b6-b549-9e71f75baaab {"id":"bfc2cf17-0ecb-47b6-b549-9e71f75baaab","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996450} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e542 bfc2cf17-0ecb-47b6-b549-9e71f75baaab /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576027857616905,"min-table-barrier-ts":449576030216912903,"admin-job-type":0} /tidb/cdc/default/default/task/position/bfc2cf17-0ecb-47b6-b549-9e71f75baaab/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b19dd0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576027857616905 MinTableBarrierTs:449576030216912903 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576027857616905, checkpointTs: 449576027857616905, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576027857616905, checkpointTs: 449576027857616905, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576027857616905, checkpointTs: 449576027857616905, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bfc2cf17-0ecb-47b6-b549-9e71f75baaab {"id":"bfc2cf17-0ecb-47b6-b549-9e71f75baaab","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996450} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e542 bfc2cf17-0ecb-47b6-b549-9e71f75baaab /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576027857616905,"min-table-barrier-ts":449576030216912903,"admin-job-type":0} /tidb/cdc/default/default/task/position/bfc2cf17-0ecb-47b6-b549-9e71f75baaab/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b19dd0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576027857616905 MinTableBarrierTs:449576030216912903 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576027857616905, checkpointTs: 449576027857616905, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576027857616905, checkpointTs: 449576027857616905, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576027857616905, checkpointTs: 449576027857616905, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bfc2cf17-0ecb-47b6-b549-9e71f75baaab {"id":"bfc2cf17-0ecb-47b6-b549-9e71f75baaab","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996450} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e542 bfc2cf17-0ecb-47b6-b549-9e71f75baaab /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576027857616905,"min-table-barrier-ts":449576030216912903,"admin-job-type":0} /tidb/cdc/default/default/task/p+ grep -q 'etcd info' osition/bfc2cf17-0ecb-47b6-b549-9e71f75baaab/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [2024/05/06 19:54:13.561 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 19:54:13.562 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2024/05/06 19:54:13.562 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=:6060] [2024/05/06 19:54:13.563 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/06 19:54:13.567 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 19:54:13.567 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449576027700592666] [resolvedTs=449576029404790788] [2024/05/06 19:54:13.568 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/06 19:54:13.568 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/06 19:54:13.568 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 19:54:13.568 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 19:54:13.568 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/06 19:54:13.571 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=4.013419ms] [2024/05/06 19:54:13.572 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 19:54:13.572 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=5.479193ms] [2024/05/06 19:54:13.575 +08:00] [INFO] [mysql.go:197] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 19:54:13.575 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:54:13.575 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:54:13.576 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:54:13.576 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:54:13.576 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:54:13.576 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:54:13.576 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:54:13.576 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:54:13.576 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:54:13.576 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:54:13.576 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:54:13.576 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:54:13.576 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:54:13.576 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:54:13.576 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:54:13.576 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:54:13.576 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 19:54:13.576 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 19:54:13.576 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 19:54:13.576 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 19:54:13.576 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 19:54:13.576 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 19:54:13.576 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 19:54:13.576 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 19:54:13.576 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 19:54:13.576 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 19:54:13.576 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 19:54:13.576 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 19:54:13.576 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 19:54:13.576 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 19:54:13.576 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 19:54:13.576 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 19:54:13.576 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 19:54:13.576 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 19:54:13.576 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/06 19:54:13.580 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 19:54:13.582 +08:00] [INFO] [mysql_ddl_sink.go:105] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/06 19:54:13.582 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449576027700592661,\"CommitTs\":449576027700592666,\"Query\":\"create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_gbk\",\"Table\":\"GBKTABLE2\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 19:54:13.582 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE"] [commitTs=449576027700592666] [namespace=] [changefeed=] [2024/05/06 19:54:13.584 +08:00] [INFO] [mysql_ddl_sink.go:125] ["Execute DDL failed, but error can be ignored"] [startTs=449576027700592661] [ddl="create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE"] [namespace=] [changefeed=] [error="Error 1050 (42S01): Table 'consistent_replicate_gbk.GBKTABLE2' already exists"] table ddl_manager.finish_mark not exists for 68-th check, retry later [2024/05/06 19:54:13.784 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449576029404790788] [2024/05/06 19:54:13.784 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449576029404790788] [2024/05/06 19:54:13.884 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=1] [appliedDDLCount=1] [currentCheckpoint=449576029404790788] [2024/05/06 19:54:13.884 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 19:54:13.884 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 19:54:13.884 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 19:54:13.884 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/06 19:54:13.884 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/06 19:54:13.884 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/06 19:54:13.884 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 19:54:13.884 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 19:54:13.884 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 19:54:13.884 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/06 19:54:13.884 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 19:54:13.884 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 19:54:13.884 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/06 19:54:13.884 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 19:54:13.884 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 19:54:13.884 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] Apply redo log successfully check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 19:54:14 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT [2024/05/06 19:54:14.385 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] table mark.finish_mark_2 not exists for 7-th check, retry later check diff failed 10-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "is-owner": true, run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully table ddl_manager.finish_mark not exists for 69-th check, retry later wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... [2024/05/06 19:54:16.396 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] check diff failed 11-th time, retry later table mark.finish_mark_2 exists table mark.finish_mark_3 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:54:16 CST 2024] <<<<<< run test case kafka_compression success! >>>>>> owner pid: 3075 owner id bfc2cf17-0ecb-47b6-b549-9e71f75baaab [Mon May 6 19:54:16 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.31973199.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37048d30000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:10853, start at 2024-05-06 19:54:14.230934698 +0800 CST m=+5.349519759 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:14.238 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:14.220 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:14.220 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37048d30000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:10853, start at 2024-05-06 19:54:14.230934698 +0800 CST m=+5.349519759 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:14.238 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:14.220 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:14.220 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37048c3c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:10933, start at 2024-05-06 19:54:14.182148106 +0800 CST m=+5.251558315 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:14.193 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:14.159 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:14.159 +0800 All versions after safe point can be accessed. (DO NOT EDIT) wait process cdc.test exit for 3-th time... Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:54:17 CST 2024] <<<<<< run test case owner_resign success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37049b08000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-5k7gg, pid:3855, start at 2024-05-06 19:54:15.114852965 +0800 CST m=+5.268262429 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:15.125 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:15.106 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:15.106 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37049b08000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-5k7gg, pid:3855, start at 2024-05-06 19:54:15.114852965 +0800 CST m=+5.268262429 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:15.125 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:15.106 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:15.106 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37049c840014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-5k7gg, pid:3935, start at 2024-05-06 19:54:15.244311438 +0800 CST m=+5.344795546 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:15.250 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:15.251 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:15.251 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table ddl_manager.finish_mark not exists for 70-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 19:54:18.408 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] table mark.finish_mark_3 not exists for 2-th check, retry later check diff failed 12-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12251.out cli tso query --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:54:20 CST 2024] <<<<<< run test case kafka_messages success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:19 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00255cb40 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576031527370764 MinTableBarrierTs:449576031527370764 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4a02f3a-0cb0-4e9b-adb7-73a35d585423 {"id":"a4a02f3a-0cb0-4e9b-adb7-73a35d585423","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996457} /tidb/cdc/default/__cdc_meta__/capture/bfc2cf17-0ecb-47b6-b549-9e71f75baaab {"id":"bfc2cf17-0ecb-47b6-b549-9e71f75baaab","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996450} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e542 bfc2cf17-0ecb-47b6-b549-9e71f75baaab /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e5a0 a4a02f3a-0cb0-4e9b-adb7-73a35d585423 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576031527370764,"min-table-barrier-ts":449576031527370764,"admin-job-type":0} /tidb/cdc/default/default/task/position/a4a02f3a-0cb0-4e9b-adb7-73a35d585423/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/bfc2cf17-0ecb-47b6-b549-9e71f75baaab/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00255cb40 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576031527370764 MinTableBarrierTs:449576031527370764 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4a02f3a-0cb0-4e9b-adb7-73a35d585423 {"id":"a4a02f3a-0cb0-4e9b-adb7-73a35d585423","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996457} /tidb/cdc/default/__cdc_meta__/capture/bfc2cf17-0ecb-47b6-b549-9e71f75baaab {"id":"bfc2cf17-0ecb-47b6-b549-9e71f75baaab","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996450} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e542 bfc2cf17-0ecb-47b6-b549-9e71f75baaab /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e5a0 a4a02f3a-0cb0-4e9b-adb7-73a35d585423 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576031527370764,"min-table-barrier-ts":449576031527370764,"admin-job-type":0} /tidb/cdc/default/default/task/position/a4a02f3a-0cb0-4e9b-adb7-73a35d585423/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/bfc2cf17-0ecb-47b6-b549-9e71f75baaab/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00255cb40 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576031527370764 MinTableBarrierTs:449576031527370764 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4a02f3a-0cb0-4e9b-adb7-73a35d585423 {"id":"a4a02f3a-0cb0-4e9b-adb7-73a35d585423","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996457} /tidb/cdc/default/__cdc_meta__/capture/bfc2cf17-0ecb-47b6-b549-9e71f75baaab {"id":"bfc2cf17-0ecb-47b6-b549-9e71f75baaab","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996450} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e542 bfc2cf17-0ecb-47b6-b549-9e71f75baaab /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e5a0 a4a02f3a-0cb0-4e9b-adb7-73a35d585423 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576031527370764,"min-table-barrier-ts":449576031527370764,"admin-job-type":0} /tidb/cdc/default/default/task/position/a4a02f3a-0cb0-4e9b-adb7-73a35d585423/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'etcd info' /tidb/cdc/default/default/task/position/bfc2cf17-0ecb-47b6-b549-9e71f75baaab/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "bfc2cf17-0ecb-47b6-b549-9e71f75baaab" | grep id =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 3-th check, retry later [2024/05/06 19:54:20.419 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] check diff failed 13-th time, retry later + set +x + tso='449576031910887425 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576031910887425 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3704c89c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc, pid:23502, start at 2024-05-06 19:54:18.025540486 +0800 CST m=+5.203239648 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:18.033 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:18.023 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:18.023 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3704c89c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc, pid:23502, start at 2024-05-06 19:54:18.025540486 +0800 CST m=+5.203239648 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:18.033 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:18.023 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:18.023 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3704cba40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc, pid:23585, start at 2024-05-06 19:54:18.258251634 +0800 CST m=+5.383336603 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:18.266 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:18.268 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:18.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/move_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/move_table/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/move_table/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } "cluster-id": "default" "id": "a4a02f3a-0cb0-4e9b-adb7-73a35d585423", "cluster-id": "default" run task successfully table ddl_manager.finish_mark not exists for 71-th check, retry later pass check, checkpoint tso not forward after 10s run task successfully wait process 8094 exit for 1-th time... wait process 8094 exit for 2-th time... wait process 8094 exit for 3-th time... wait process 8094 exit for 4-th time... wait process 8094 exit for 5-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8094) - No such process wait process 8094 exit for 6-th time... process 8094 already exit [Mon May 6 19:54:16 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.88108812.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:19 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365853277194306638 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 19:52:56.392703665 +0800 CST StartTs:449576010069835778 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0030f86c0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576010082942983} {CheckpointTs:449576014133067790 MinTableBarrierTs:449576031670763524 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449576014133067790, checkpointTs: 449576014133067790, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449576014133067790, checkpointTs: 449576014133067790, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576014133067790, checkpointTs: 449576014133067790, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576014133067790, checkpointTs: 449576014133067790, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e436f59a-c3a2-4a44-9b88-e45756e7b7fd {"id":"e436f59a-c3a2-4a44-9b88-e45756e7b7fd","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996456} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbfa75902 e436f59a-c3a2-4a44-9b88-e45756e7b7fd /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365853277194306638,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T19:52:56.392703665+08:00","start-ts":449576010069835778,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576010082942983} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449576014133067790,"min-table-barrier-ts":449576031670763524,"admin-job-type":0} /tidb/cdc/default/default/task/position/e436f59a-c3a2-4a44-9b88-e45756e7b7fd/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853277194306638 {"id":7365853277194306638,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365853277194306638 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 19:52:56.392703665 +0800 CST StartTs:449576010069835778 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0030f86c0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576010082942983} {CheckpointTs:449576014133067790 MinTableBarrierTs:449576031670763524 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449576014133067790, checkpointTs: 449576014133067790, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449576014133067790, checkpointTs: 449576014133067790, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576014133067790, checkpointTs: 449576014133067790, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576014133067790, checkpointTs: 449576014133067790, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e436f59a-c3a2-4a44-9b88-e45756e7b7fd {"id":"e436f59a-c3a2-4a44-9b88-e45756e7b7fd","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996456} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbfa75902 e436f59a-c3a2-4a44-9b88-e45756e7b7fd /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365853277194306638,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T19:52:56.392703665+08:00","start-ts":449576010069835778,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576010082942983} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449576014133067790,"min-table-barrier-ts":449576031670763524,"admin-job-type":0} /tidb/cdc/default/default/task/position/e436f59a-c3a2-4a44-9b88-e45756e7b7fd/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853277194306638 {"id":7365853277194306638,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365853277194306638 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 19:52:56.392703665 +0800 CST StartTs:449576010069835778 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0030f86c0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576010082942983} {CheckpointTs:449576014133067790 MinTableBarrierTs:449576031670763524 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449576014133067790, checkpointTs: 449576014133067790, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449576014133067790, checkpointTs: 449576014133067790, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576014133067790, checkpointTs: 449576014133067790, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576014133067790, checkpointTs: 449576014133067790, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e436f59a-c3a2-4a44-9b88-e45756e7b7fd {"id":"e436f59a-c3a2-4a44-9b88-e45756e7b7fd","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996456} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dbfa75902 e436f59a-c3a2-4a44-9b88-e45756e7b7fd /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365853277194306638,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T19:52:56.392703665+08:00","start-ts":449576010069835778,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576010082942983} /tidb/cdc/default/default/changefeed/status/ddl-only-block-relat+ grep -q 'etcd info' ed-table {"checkpoint-ts":449576014133067790,"min-table-barrier-ts":449576031670763524,"admin-job-type":0} /tidb/cdc/default/default/task/position/e436f59a-c3a2-4a44-9b88-e45756e7b7fd/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853277194306638 {"id":7365853277194306638,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check diff successfully check_ts_forward ddl-only-block-related-table changefeed is working normally rts: 449576014133067800->449576032194789400 checkpoint: 449576014133067800->449576032194789400 run task successfully wait process cdc.test exit for 1-th time... [Mon May 6 19:54:22 CST 2024] <<<<<< START cdc server in batch_update_to_no_batch case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.52075209.out server --log-file /tmp/tidb_cdc_test/batch_update_to_no_batch/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_update_to_no_batch/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 19:54:22 CST 2024] <<<<<< START cdc server in cli_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.1230512307.out server --log-file /tmp/tidb_cdc_test/cli_with_auth/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_with_auth/cdc_data --cluster-id default --config /tmp/tidb_cdc_test/cli_with_auth/server.toml ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 capture_id: a4a02f3a-0cb0-4e9b-adb7-73a35d585423 ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep 'a4a02f3a-0cb0-4e9b-adb7-73a35d585423' a4a02f3a-0cb0-4e9b-adb7-73a35d585423 run task successfully test_hang_up_owner: pass build mysql_test successfully run all mysql test cases time="2024-05-06T19:54:18+08:00" level=info msg="running tests: [alter_table bigint composite_index concurrent_ddl date_formats drop mysql_replace]" time="2024-05-06T19:54:18+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0001a6000 {{} {} 0} {0 0} [0xc0001c0000] map[] 0 1 0xc0001aa000 false map[0xc0001c0000:map[0xc0001c0000:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark exists [2024/05/06 19:54:22.432 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] check diff failed 14-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... check diff successfully Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 4-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 3-th time... [2024/05/06 19:54:16.730 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 32"] wait process cdc.test exit for 1-th time... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.24849.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:54:23 CST 2024] <<<<<< run test case ddl_only_block_related_table success! >>>>>> wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 3-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:54:23 CST 2024] <<<<<< run test case kafka_sink_error_resume success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... wait process cdc.test exit for 3-th time... [2024/05/06 19:54:24.444 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] [2024/05/06 19:54:24.458 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark] [ts=449576032671105029] [2024/05/06 19:54:24.458 +08:00] [INFO] [case.go:498] ["all tables synced"] [endTs=449576032671105029] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit run test case test_expire_owner [Mon May 6 19:54:24 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.33723374.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_owner.server1 --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 5-th check, retry later wait process cdc.test exit for 4-th time... + set +x + tso='449576032920666113 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576032920666113 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x check diff failed 15-th time, retry later wait process cdc.test exit for 5-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:25 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d37b0ec7-c0f7-41b5-a094-c5ce5453ce7c {"id":"d37b0ec7-c0f7-41b5-a094-c5ce5453ce7c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996462} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1004339 d37b0ec7-c0f7-41b5-a094-c5ce5453ce7c /tidb/cdc/default/default/upstream/7365853648514039232 {"id":7365853648514039232,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d37b0ec7-c0f7-41b5-a094-c5ce5453ce7c {"id":"d37b0ec7-c0f7-41b5-a094-c5ce5453ce7c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996462} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1004339 d37b0ec7-c0f7-41b5-a094-c5ce5453ce7c /tidb/cdc/default/default/upstream/7365853648514039232 {"id":7365853648514039232,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d37b0ec7-c0f7-41b5-a094-c5ce5453ce7c {"id":"d37b0ec7-c0f7-41b5-a094-c5ce5453ce7c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996462} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1004339 d37b0ec7-c0f7-41b5-a094-c5ce5453ce7c /tidb/cdc/default/default/upstream/7365853648514039232 {"id":7365853648514039232,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5263.out cli changefeed create '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=true' -c test + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:25 GMT < Content-Length: 871 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1631f9de-e001-4697-bcbd-07314e03b802 {"id":"1631f9de-e001-4697-bcbd-07314e03b802","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996462} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc0fc6311 1631f9de-e001-4697-bcbd-07314e03b802 /tidb/cdc/default/default/upstream/7365853647614662724 {"id":7365853647614662724,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1631f9de-e001-4697-bcbd-07314e03b802 {"id":"1631f9de-e001-4697-bcbd-07314e03b802","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996462} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc0fc6311 1631f9de-e001-4697-bcbd-07314e03b802 /tidb/cdc/default/default/upstream/7365853647614662724 {"id":7365853647614662724,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1631f9de-e001-4697-bcbd-07314e03b802 {"id":"1631f9de-e001-4697-bcbd-07314e03b802","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996462} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc0fc6311 1631f9de-e001-4697-bcbd-07314e03b802 /tidb/cdc/default/default/upstream/7365853647614662724 {"id":7365853647614662724,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12372.out cli changefeed create --start-ts=449576031910887425 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365853647614662724,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:54:25.705974504+08:00","start_ts":449576031910887425,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576031910887425,"checkpoint_ts":449576031910887425,"checkpoint_time":"2024-05-06 19:54:19.621"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... Create changefeed successfully! ID: test Info: {"upstream_id":7365853648514039232,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=true","create_time":"2024-05-06T19:54:25.755774047+08:00","start_ts":449576033492926468,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576033492926468,"checkpoint_ts":449576033492926468,"checkpoint_time":"2024-05-06 19:54:25.656"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... ***************** properties ***************** "updateproportion"="0" "mysql.db"="move_table" "recordcount"="10000" "requestdistribution"="uniform" "scanproportion"="0" "threadcount"="10" "mysql.user"="root" "dotransactions"="false" "workload"="core" "readallfields"="true" "operationcount"="0" "mysql.port"="4000" "readproportion"="0" "insertproportion"="0" "mysql.host"="127.0.0.1" ********************************************** cdc.test: no process found wait process cdc.test exit for 6-th time... process cdc.test already exit [Mon May 6 19:54:25 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:54:26 CST 2024] <<<<<< run test case mq_sink_lost_callback success! >>>>>> table mark.finish_mark_3 not exists for 6-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x table test.simple exists table test.`simple-dash` not exists for 1-th check, retry later + set +x [2024/05/06 19:54:27.298 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark1000] [2024/05/06 19:54:27.298 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark1000] check diff failed 16-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_nfs Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:27 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0039add40 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576032576208906 MinTableBarrierTs:449576033886928902 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576032576208906, checkpointTs: 449576032576208906, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576032576208906, checkpointTs: 449576032576208906, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576032576208906, checkpointTs: 449576032576208906, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a12f0e90-2113-4856-b00d-d34aabc71a27 {"id":"a12f0e90-2113-4856-b00d-d34aabc71a27","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996464} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e5ef a12f0e90-2113-4856-b00d-d34aabc71a27 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576032576208906,"min-table-barrier-ts":449576033886928902,"admin-job-type":0} /tidb/cdc/default/default/task/position/a12f0e90-2113-4856-b00d-d34aabc71a27/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0039add40 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576032576208906 MinTableBarrierTs:449576033886928902 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576032576208906, checkpointTs: 449576032576208906, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576032576208906, checkpointTs: 449576032576208906, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576032576208906, checkpointTs: 449576032576208906, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a12f0e90-2113-4856-b00d-d34aabc71a27 {"id":"a12f0e90-2113-4856-b00d-d34aabc71a27","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996464} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e5ef a12f0e90-2113-4856-b00d-d34aabc71a27 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576032576208906,"min-table-barrier-ts":449576033886928902,"admin-job-type":0} /tidb/cdc/default/default/task/position/a12f0e90-2113-4856-b00d-d34aabc71a27/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0039add40 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576032576208906 MinTableBarrierTs:449576033886928902 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576032576208906, checkpointTs: 449576032576208906, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576032576208906, checkpointTs: 449576032576208906, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576032576208906, checkpointTs: 449576032576208906, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a12f0e90-2113-4856-b00d-d34aabc71a27 {"id":"a12f0e90-2113-4856-b00d-d34aabc71a27","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996464} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e5ef a12f0e90-2113-4856-b00d-d34aabc71a27 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576032576208906,"min-table-barrier-ts":449576033886928902,"admin-job-type":0} /tidb/cdc/default/default/task/position/a12f0e90-2113-4856-b00d-d34aabc71a27/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'etcd info' /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' table mark.finish_mark_3 not exists for 7-th check, retry later Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 17-th time, retry later Run finished, takes 2.780516029s INSERT - Takes(s): 2.8, Count: 10000, OPS: 3603.2, Avg(us): 2741, Min(us): 814, Max(us): 801661, 95th(us): 2000, 99th(us): 41000 [Mon May 6 19:54:28 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2493924941.out server --log-file /tmp/tidb_cdc_test/move_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 "is-owner": true, run task successfully table test.`simple-dash` not exists for 2-th check, retry later [2024/05/06 19:54:29.299 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark1000] [2024/05/06 19:54:29.371 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark2000] [2024/05/06 19:54:29.371 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark2000] Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 8-th check, retry later owner pid: 3377 owner id a12f0e90-2113-4856-b00d-d34aabc71a27 process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v 'a12f0e90-2113-4856-b00d-d34aabc71a27' /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e5ef run task successfully process status: S cdc.test cli capture list 2>&1 | grep '"is-owner": true' start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:54:30 CST 2024] <<<<<< run test case mq_sink_dispatcher success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365853647614662724, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449576034872852487, "checkpoint_time": "2024-05-06 19:54:30.920", "error": null }' + echo '{ "upstream_id": 7365853647614662724, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449576034872852487, "checkpoint_time": "2024-05-06 19:54:30.920", "error": null }' { "upstream_id": 7365853647614662724, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449576034872852487, "checkpoint_time": "2024-05-06 19:54:30.920", "error": null } ++ echo '{' '"upstream_id":' 7365853647614662724, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449576034872852487, '"checkpoint_time":' '"2024-05-06' '19:54:30.920",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365853647614662724, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449576034872852487, '"checkpoint_time":' '"2024-05-06' '19:54:30.920",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 18-th time, retry later [2024/05/06 19:54:31.360 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark1000] [2024/05/06 19:54:31.371 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark2000] [2024/05/06 19:54:31.387 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark1000] [ts=449576034489860101] [2024/05/06 19:54:31.387 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark1000] [2024/05/06 19:54:31.387 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576034489860101] [2024/05/06 19:54:31.390 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:31.462 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:54:31.467 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:31.472 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 "is-owner": true, run task successfully test_expire_owner pass changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:31 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/671896cf-401e-4092-bd9a-9160eb98eddc {"id":"671896cf-401e-4092-bd9a-9160eb98eddc","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996469} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc107f35c 671896cf-401e-4092-bd9a-9160eb98eddc /tidb/cdc/default/default/upstream/7365853663926291761 {"id":7365853663926291761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/671896cf-401e-4092-bd9a-9160eb98eddc {"id":"671896cf-401e-4092-bd9a-9160eb98eddc","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996469} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc107f35c 671896cf-401e-4092-bd9a-9160eb98eddc /tidb/cdc/default/default/upstream/7365853663926291761 {"id":7365853663926291761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/671896cf-401e-4092-bd9a-9160eb98eddc {"id":"671896cf-401e-4092-bd9a-9160eb98eddc","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996469} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc107f35c 671896cf-401e-4092-bd9a-9160eb98eddc /tidb/cdc/default/default/upstream/7365853663926291761 {"id":7365853663926291761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.25000.out cli changefeed create --start-ts=449576032920666113 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' Create changefeed successfully! ID: dc0341b5-a12c-42df-a8ff-2e511d84a438 Info: {"upstream_id":7365853663926291761,"namespace":"default","id":"dc0341b5-a12c-42df-a8ff-2e511d84a438","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:54:32.324002281+08:00","start_ts":449576032920666113,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576032920666113,"checkpoint_ts":449576032920666113,"checkpoint_time":"2024-05-06 19:54:23.473"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5311.out cli changefeed pause -c test [2024/05/06 19:54:32.572 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark3000] [2024/05/06 19:54:32.573 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark3000] wait process cdc.test exit for 1-th time... Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 9-th check, retry later PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12764.out cli changefeed --changefeed-id custom-changefeed-name pause wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:54:33 CST 2024] <<<<<< run test case kafka_column_selector success! >>>>>> pass test: get status pass test: health pass test: changefeed apis pass test: delete changefeed apis pass test: capture apis pass test: processor apis pass test: owner apis pass test: set log level + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit ++ date + echo '[Mon May 6 19:54:32 CST 2024] <<<<<< run test case api_v2 success! >>>>>>' [Mon May 6 19:54:32 CST 2024] <<<<<< run test case api_v2 success! >>>>>> + stop_tidb_cluster PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:54:33.389 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark2000] [2024/05/06 19:54:33.563 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark2000] [ts=449576035433578502] [2024/05/06 19:54:33.563 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark2000] [2024/05/06 19:54:33.563 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576035433578502] [2024/05/06 19:54:33.566 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:33.570 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] check diff failed 19-th time, retry later wait process cdc.test exit for 3-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_auto_stop/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + set +x [Mon May 6 19:54:33 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2503625038.out server --log-file /tmp/tidb_cdc_test/move_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:54:33.660 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:33.666 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit run test case test_owner_cleanup_stale_tasks [Mon May 6 19:54:34 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.35683570.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5346.out cli changefeed update -c test '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=false' --no-confirm ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Diff of changefeed config: {Type:update Path:[SinkURI] From:mysql://root@127.0.0.1:3306/?batch-dml-enable=true To:mysql://root@127.0.0.1:3306/?batch-dml-enable=false} [2024/05/06 19:54:34.573 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark3000] + set +x table mark.finish_mark_3 not exists for 10-th check, retry later Update changefeed config successfully! ID: test Info: {"upstream_id":7365853648514039232,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=false","create_time":"2024-05-06T19:54:25.755774047+08:00","start_ts":449576033492926468,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":0,"checkpoint_ts":449576034305572892,"checkpoint_time":"2024-05-06 19:54:28.756"} PASS coverage: 2.3% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3705c3c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-rqt30, pid:4068, start at 2024-05-06 19:54:34.135056017 +0800 CST m=+5.148291914 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:34.141 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:34.146 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:34.146 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3705c3c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-rqt30, pid:4068, start at 2024-05-06 19:54:34.135056017 +0800 CST m=+5.148291914 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:34.141 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:34.146 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:34.146 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3705c3980019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-rqt30, pid:4156, start at 2024-05-06 19:54:34.125640363 +0800 CST m=+5.084916805 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:34.133 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:34.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:34.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 20-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:54:35.866 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark4000] [2024/05/06 19:54:35.866 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark4000] + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5391.out cli changefeed resume -c test [2024/05/06 19:54:36.671 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark3000] [2024/05/06 19:54:36.687 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark3000] [ts=449576036351082500] [2024/05/06 19:54:36.687 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark3000] [2024/05/06 19:54:36.687 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576036351082500] [2024/05/06 19:54:36.689 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:36.692 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:54:36.696 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:36.701 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_column_selector_avro/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:54:36 CST 2024] <<<<<< run test case kafka_column_selector_avro success! >>>>>> table mark.finish_mark_3 not exists for 11-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:36 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/dc0341b5-a12c-42df-a8ff-2e511d84a438 {UpstreamID:7365853663926291761 Namespace:default ID:dc0341b5-a12c-42df-a8ff-2e511d84a438 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:54:32.324002281 +0800 CST StartTs:449576032920666113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00143d9e0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576035240640513} {CheckpointTs:449576033589133482 MinTableBarrierTs:449576036263002123 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576036263002123, checkpointTs: 449576033589133695, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/671896cf-401e-4092-bd9a-9160eb98eddc {"id":"671896cf-401e-4092-bd9a-9160eb98eddc","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996469} /tidb/cdc/default/__cdc_meta__/capture/afbc0a4a-139f-4d5d-a458-05d51fbedd73 {"id":"afbc0a4a-139f-4d5d-a458-05d51fbedd73","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996474} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc107f35c 671896cf-401e-4092-bd9a-9160eb98eddc /tidb/cdc/default/__cdc_meta__/owner/22318f4dc107f3b7 afbc0a4a-139f-4d5d-a458-05d51fbedd73 /tidb/cdc/default/default/changefeed/info/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"upstream-id":7365853663926291761,"namespace":"default","changefeed-id":"dc0341b5-a12c-42df-a8ff-2e511d84a438","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:54:32.324002281+08:00","start-ts":449576032920666113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576035240640513} /tidb/cdc/default/default/changefeed/status/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":449576033589133482,"min-table-barrier-ts":449576036263002123,"admin-job-type":0} /tidb/cdc/default/default/task/position/671896cf-401e-4092-bd9a-9160eb98eddc/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/afbc0a4a-139f-4d5d-a458-05d51fbedd73/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853663926291761 {"id":7365853663926291761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/dc0341b5-a12c-42df-a8ff-2e511d84a438 {UpstreamID:7365853663926291761 Namespace:default ID:dc0341b5-a12c-42df-a8ff-2e511d84a438 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:54:32.324002281 +0800 CST StartTs:449576032920666113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00143d9e0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576035240640513} {CheckpointTs:449576033589133482 MinTableBarrierTs:449576036263002123 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576036263002123, checkpointTs: 449576033589133695, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/671896cf-401e-4092-bd9a-9160eb98eddc {"id":"671896cf-401e-4092-bd9a-9160eb98eddc","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996469} /tidb/cdc/default/__cdc_meta__/capture/afbc0a4a-139f-4d5d-a458-05d51fbedd73 {"id":"afbc0a4a-139f-4d5d-a458-05d51fbedd73","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996474} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc107f35c 671896cf-401e-4092-bd9a-9160eb98eddc /tidb/cdc/default/__cdc_meta__/owner/22318f4dc107f3b7 afbc0a4a-139f-4d5d-a458-05d51fbedd73 /tidb/cdc/default/default/changefeed/info/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"upstream-id":7365853663926291761,"namespace":"default","changefeed-id":"dc0341b5-a12c-42df-a8ff-2e511d84a438","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:54:32.324002281+08:00","start-ts":449576032920666113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576035240640513} /tidb/cdc/default/default/changefeed/status/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":44957603+ grep -q 'failed to get info:' 3589133482,"min-table-barrier-ts":449576036263002123,"admin-job-type":0} /tidb/cdc/default/default/task/position/671896cf-401e-4092-bd9a-9160eb98eddc/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/afbc0a4a-139f-4d5d-a458-05d51fbedd73/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853663926291761 {"id":7365853663926291761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/dc0341b5-a12c-42df-a8ff-2e511d84a438 {UpstreamID:7365853663926291761 Namespace:default ID:dc0341b5-a12c-42df-a8ff-2e511d84a438 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:54:32.324002281 +0800 CST StartTs:449576032920666113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00143d9e0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576035240640513} {CheckpointTs:449576033589133482 MinTableBarrierTs:449576036263002123 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576036263002123, checkpointTs: 449576033589133695, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/671896cf-401e-4092-bd9a-9160eb98eddc {"id":"671896cf-401e-4092-bd9a-9160eb98eddc","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996469} /tidb/cdc/default/__cdc_meta__/capture/afbc0a4a-139f-4d5d-a458-05d51fbedd73 {"id":"afbc0a4a-139f-4d5d-a458-05d51fbedd73","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996474} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc107f35c 671896cf-401e-4092-bd9a-9160eb98eddc /tidb/cdc/default/__cdc_meta__/owner/22318f4dc107f3b7 afbc0a4a-139f-4d5d-a458-05d51fbedd73 /tidb/cdc/default/default/changefeed/info/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"upstream-id":7365853663926291761,"namespace":"default","changefeed-id":"dc0341b5-a12c-42df-a8ff-2e511d84a438","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:54:32.324002281+08:00","start-ts":449576032920666113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576035240640513} /tidb/cdc/default/default/changefeed/status/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":449576033589133482,"min-table-barrier-ts":449576036263002123,"admin-job-type":0} /tidb/cdc/default/default/task/position/671896cf-401e-4092-bd9a-9160eb98eddc/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/afbc0a4a-139f-4d5d-a458-05d51fbedd73/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853663926291761 {"id":7365853663926291761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:54:36 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2509925101.out server --log-file /tmp/tidb_cdc_test/move_table/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data3 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 PASS coverage: 2.3% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:37 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004092a20 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576034948612097 MinTableBarrierTs:449576036259332098 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576034948612097, checkpointTs: 449576034948612097, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576034948612097, checkpointTs: 449576034948612097, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576034948612097, checkpointTs: 449576034948612097, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b {"id":"6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996474} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e662 6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576034948612097,"min-table-barrier-ts":449576036521476098,"admin-job-type":0} /tidb/cdc/default/default/task/position/6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004092a20 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576034948612097 MinTableBarrierTs:449576036259332098 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576034948612097, checkpointTs: 449576034948612097, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576034948612097, checkpointTs: 449576034948612097, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576034948612097, checkpointTs: 449576034948612097, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b {"id":"6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996474} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e662 6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576034948612097,"min-table-barrier-ts":449576036521476098,"admin-job-type":0} /tidb/cdc/default/default/task/p+ grep -q 'failed to get info:' osition/6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004092a20 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576034948612097 MinTableBarrierTs:449576036259332098 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576034948612097, checkpointTs: 449576034948612097, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576034948612097, checkpointTs: 449576034948612097, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576034948612097, checkpointTs: 449576034948612097, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b {"id":"6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996474} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e662 6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576034948612097,"min-table-barrier-ts":449576036521476098,"admin-job-type":0} /tidb/cdc/default/default/task/position/6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' ./t/alter_table.test: ok! 64 test cases passed, take time 17.913211631 s time="2024-05-06T19:54:37+08:00" level=info msg="run test [alter_table] ok" time="2024-05-06T19:54:37+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc00030c2c0 {{} {} 0} {0 0} [0xc00038c000] map[] 0 1 0xc000300ae0 false map[0xc00038c000:map[0xc00038c000:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 21-th time, retry later [2024/05/06 19:54:37.867 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365853647614662724, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449576035397140486, "checkpoint_time": "2024-05-06 19:54:32.920", "error": null }' + echo '{ "upstream_id": 7365853647614662724, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449576035397140486, "checkpoint_time": "2024-05-06 19:54:32.920", "error": null }' { "upstream_id": 7365853647614662724, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449576035397140486, "checkpoint_time": "2024-05-06 19:54:32.920", "error": null } ++ echo '{' '"upstream_id":' 7365853647614662724, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449576035397140486, '"checkpoint_time":' '"2024-05-06' '19:54:32.920",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365853647614662724, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449576035397140486, '"checkpoint_time":' '"2024-05-06' '19:54:32.920",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12866.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Mon May 6 19:54:38 CST 2024] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.54725474.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} table mark.finish_mark_3 not exists for 12-th check, retry later "is-owner": true, run task successfully Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7365853647614662724,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:54:25.705974504+08:00","start_ts":449576031910887425,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":0,"checkpoint_ts":449576035397140486,"checkpoint_time":"2024-05-06 19:54:32.920"} PASS coverage: 3.0% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:54:38.973 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark5000] [2024/05/06 19:54:38.973 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark5000] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3706041c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:7218, start at 2024-05-06 19:54:38.253878907 +0800 CST m=+5.304019084 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:38.262 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:38.265 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:38.265 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 22-th time, retry later [2024/05/06 19:54:39.963 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/debezium/run.sh using Sink-Type: mysql... <<================= =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:54:40 CST 2024] <<<<<< run test case lossy_ddl success! >>>>>> owner pid: 3573 owner id 6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b [Mon May 6 19:54:40 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.36833685.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server2 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:40 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/dc0341b5-a12c-42df-a8ff-2e511d84a438 {UpstreamID:7365853663926291761 Namespace:default ID:dc0341b5-a12c-42df-a8ff-2e511d84a438 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:54:32.324002281 +0800 CST StartTs:449576032920666113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00148bb90 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576035240640513} {CheckpointTs:449576033956135260 MinTableBarrierTs:449576037075648513 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2df0df91-4b4f-4f75-8268-545024bdcf73 {"id":"2df0df91-4b4f-4f75-8268-545024bdcf73","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996478} /tidb/cdc/default/__cdc_meta__/capture/671896cf-401e-4092-bd9a-9160eb98eddc {"id":"671896cf-401e-4092-bd9a-9160eb98eddc","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996469} /tidb/cdc/default/__cdc_meta__/capture/afbc0a4a-139f-4d5d-a458-05d51fbedd73 {"id":"afbc0a4a-139f-4d5d-a458-05d51fbedd73","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996474} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc107f35c 671896cf-401e-4092-bd9a-9160eb98eddc /tidb/cdc/default/__cdc_meta__/owner/22318f4dc107f3b7 afbc0a4a-139f-4d5d-a458-05d51fbedd73 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc107f3f0 2df0df91-4b4f-4f75-8268-545024bdcf73 /tidb/cdc/default/default/changefeed/info/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"upstream-id":7365853663926291761,"namespace":"default","changefeed-id":"dc0341b5-a12c-42df-a8ff-2e511d84a438","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:54:32.324002281+08:00","start-ts":449576032920666113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576035240640513} /tidb/cdc/default/default/changefeed/status/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":449576033956135260,"min-table-barrier-ts":449576037075648513,"admin-job-type":0} /tidb/cdc/default/default/task/position/2df0df91-4b4f-4f75-8268-545024bdcf73/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/671896cf-401e-4092-bd9a-9160eb98eddc/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/afbc0a4a-139f-4d5d-a458-05d51fbedd73/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853663926291761 {"id":7365853663926291761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/dc0341b5-a12c-42df-a8ff-2e511d84a438 {UpstreamID:7365853663926291761 Namespace:default ID:dc0341b5-a12c-42df-a8ff-2e511d84a438 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:54:32.324002281 +0800 CST StartTs:449576032920666113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00148bb90 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576035240640513} {CheckpointTs:449576033956135260 MinTableBarrierTs:449576037075648513 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2df0df91-4b4f-4f75-8268-545024bdcf73 {"id":"2df0df91-4b4f-4f75-8268-545024bdcf73","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996478} /tidb/cdc/default/__cdc_meta__/capture/671896cf-401e-4092-bd9a-9160eb98eddc {"id":"671896cf-401e-4092-bd9a-9160eb98eddc","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996469} /tidb/cdc/default/__cdc_meta__/capture/afbc0a4a-139f-4d5d-a458-05d51fbedd73 {"id":"afbc0a4a-139f-4d5d-a458-05d51fbedd73","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996474} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc107f35c 671896cf-401e-4092-bd9a-9160eb98eddc /tidb/cdc/default/__cdc_meta__/owner/22318f4dc107f3b7 afbc0a4a-139f-4d5d-a458-05d51fbedd73 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc107f3f0 2df0df91-4b4f-4f75-8268-545024bdcf73 /tidb/cdc/default/default/changefeed/info/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"upstream-id":7365853663926291761,"namespace":"default","changefeed-id":"dc0341b5-a12c-42df-a8ff-2e511d84a438","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:54:32.324002281+08:00","start-ts":449576032920666113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"sync+ grep -q 'failed to get info:' ed-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576035240640513} /tidb/cdc/default/default/changefeed/status/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":449576033956135260,"min-table-barrier-ts":449576037075648513,"admin-job-type":0} /tidb/cdc/default/default/task/position/2df0df91-4b4f-4f75-8268-545024bdcf73/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/671896cf-401e-4092-bd9a-9160eb98eddc/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/afbc0a4a-139f-4d5d-a458-05d51fbedd73/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853663926291761 {"id":7365853663926291761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/dc0341b5-a12c-42df-a8ff-2e511d84a438 {UpstreamID:7365853663926291761 Namespace:default ID:dc0341b5-a12c-42df-a8ff-2e511d84a438 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:54:32.324002281 +0800 CST StartTs:449576032920666113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00148bb90 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576035240640513} {CheckpointTs:449576033956135260 MinTableBarrierTs:449576037075648513 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2df0df91-4b4f-4f75-8268-545024bdcf73 {"id":"2df0df91-4b4f-4f75-8268-545024bdcf73","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996478} /tidb/cdc/default/__cdc_meta__/capture/671896cf-401e-4092-bd9a-9160eb98eddc {"id":"671896cf-401e-4092-bd9a-9160eb98eddc","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996469} /tidb/cdc/default/__cdc_meta__/capture/afbc0a4a-139f-4d5d-a458-05d51fbedd73 {"id":"afbc0a4a-139f-4d5d-a458-05d51fbedd73","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996474} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc107f35c 671896cf-401e-4092-bd9a-9160eb98eddc /tidb/cdc/default/__cdc_meta__/owner/22318f4dc107f3b7 afbc0a4a-139f-4d5d-a458-05d51fbedd73 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc107f3f0 2df0df91-4b4f-4f75-8268-545024bdcf73 /tidb/cdc/default/default/changefeed/info/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"upstream-id":7365853663926291761,"namespace":"default","changefeed-id":"dc0341b5-a12c-42df-a8ff-2e511d84a438","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:54:32.324002281+08:00","start-ts":449576032920666113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576035240640513} /tidb/cdc/default/default/changefeed/status/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":449576033956135260,"min-table-barrier-ts":449576037075648513,"admin-job-type":0} /tidb/cdc/default/default/task/position/2df0df91-4b4f-4f75-8268-545024bdcf73/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/671896cf-401e-4092-bd9a-9160eb98eddc/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/afbc0a4a-139f-4d5d-a458-05d51fbedd73/dc0341b5-a12c-42df-a8ff-2e511d84a438 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853663926291761 {"id":7365853663926291761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12902.out cli changefeed --changefeed-id custom-changefeed-name resume table mark.finish_mark_3 not exists for 13-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: mysql... <<================= table move_table.usertable exists go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/apache/pulsar-client-go v0.11.0 [2024/05/06 19:54:40.973 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/stretchr/objx v0.5.2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3706041c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:7218, start at 2024-05-06 19:54:38.253878907 +0800 CST m=+5.304019084 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:38.262 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:38.265 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:38.265 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370605880018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:7298, start at 2024-05-06 19:54:38.353210263 +0800 CST m=+5.353385931 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:38.363 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:38.356 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:38.356 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5436.out cli changefeed pause -c test PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... PASS + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 Logging trace to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } coverage: 2.3% of statements in github.com/pingcap/tiflow/... < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:41 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/98349aa6-9eeb-49cd-8512-7802465f84b4 {"id":"98349aa6-9eeb-49cd-8512-7802465f84b4","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996478} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1452a69 98349aa6-9eeb-49cd-8512-7802465f84b4 /tidb/cdc/default/default/upstream/7365853734163437032 {"id":7365853734163437032,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/98349aa6-9eeb-49cd-8512-7802465f84b4 {"id":"98349aa6-9eeb-49cd-8512-7802465f84b4","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996478} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1452a69 98349aa6-9eeb-49cd-8512-7802465f84b4 /tidb/cdc/default/default/upstream/7365853734163437032 {"id":7365853734163437032,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/98349aa6-9eeb-49cd-8512-7802465f84b4 {"id":"98349aa6-9eeb-49cd-8512-7802465f84b4","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996478} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1452a69 98349aa6-9eeb-49cd-8512-7802465f84b4 /tidb/cdc/default/default/upstream/7365853734163437032 {"id":7365853734163437032,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.5531.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37062b7c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-dwwvr, pid:4236, start at 2024-05-06 19:54:40.779784251 +0800 CST m=+5.306626354 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:40.790 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:40.786 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:40.786 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/aws/smithy-go v1.13.5 check diff failed 23-th time, retry later Create changefeed successfully! ID: f31dd4c7-5d04-4a74-b228-1813ee650c5a Info: {"upstream_id":7365853734163437032,"namespace":"default","id":"f31dd4c7-5d04-4a74-b228-1813ee650c5a","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:54:41.74766651+08:00","start_ts":449576037684609026,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576037684609026,"checkpoint_ts":449576037684609026,"checkpoint_time":"2024-05-06 19:54:41.646"} PASS [2024/05/06 19:54:41.872 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark6000] [2024/05/06 19:54:41.872 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark6000] go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/hashicorp/errwrap v1.0.0 coverage: 2.6% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:54:41.977 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] [2024/05/06 19:54:41.994 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark4000] [ts=449576037294800901] [2024/05/06 19:54:41.994 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark4000] [2024/05/06 19:54:41.994 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576037294800901] [2024/05/06 19:54:42.063 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:42.066 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:54:42.070 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:42.075 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/go-playground/locales v0.14.1 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 14-th check, retry later The 1 times to try to start tidb cluster... + set +x [2024/05/06 19:54:43.074 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5468.out cli changefeed update -c test '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=true' --no-confirm Diff of changefeed config: {Type:update Path:[SinkURI] From:mysql://root@127.0.0.1:3306/?batch-dml-enable=false To:mysql://root@127.0.0.1:3306/?batch-dml-enable=true} [Mon May 6 19:54:43 CST 2024] <<<<<< START cdc server in consistent_replicate_nfs case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_nfs.85518553.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Update changefeed config successfully! ID: test Info: {"upstream_id":7365853648514039232,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=true","create_time":"2024-05-06T19:54:25.755774047+08:00","start_ts":449576033492926468,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":0,"checkpoint_ts":449576034436644892,"checkpoint_time":"2024-05-06 19:54:29.256"} PASS coverage: 2.3% of statements in github.com/pingcap/tiflow/... + set +x table batch_add_table.finish_mark not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_api_tls_with_user_auth/run.sh using Sink-Type: mysql... <<================= WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Requirement already up-to-date: requests==2.26.0 in /usr/local/lib/python3.6/site-packages Requirement already up-to-date: certifi>=2017.4.17 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: idna<4,>=2.5; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:43 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001acae10 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576037832196099 MinTableBarrierTs:449576038094340100 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03eb6b2c-343b-49c5-bc30-3b734541a901 {"id":"03eb6b2c-343b-49c5-bc30-3b734541a901","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996480} /tidb/cdc/default/__cdc_meta__/capture/6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b {"id":"6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996474} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e662 6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e6ad 03eb6b2c-343b-49c5-bc30-3b734541a901 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576037832196099,"min-table-barrier-ts":449576038094340100,"admin-job-type":0} /tidb/cdc/default/default/task/position/03eb6b2c-343b-49c5-bc30-3b734541a901/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001acae10 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576037832196099 MinTableBarrierTs:449576038094340100 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03eb6b2c-343b-49c5-bc30-3b734541a901 {"id":"03eb6b2c-343b-49c5-bc30-3b734541a901","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996480} /tidb/cdc/default/__cdc_meta__/capture/6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b {"id":"6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996474} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e662 6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e6ad 03eb6b2c-343b-49c5-bc30-3b734541a901 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576037832196099,"min-table-barrier-ts":449576038094340100,"admin-job-type":0} /tidb/cdc/default/default/task/position/03eb6b2c-343b-49c5-bc30-3b734541a901/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001acae10 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576037832196099 MinTableBarrierTs:449576038094340100 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03eb6b2c-343b-49c5-bc30-3b734541a901 {"id":"03eb6b2c-343b-49c5-bc30-3b734541a901","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996480} /tidb/cdc/default/__cdc_meta__/capture/6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b {"id":"6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996474} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e662 6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e6ad 03eb6b2c-343b-49c5-bc30-3b734541a901 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576037832196099,"min-table-barrier-ts":449576038094340100,"admin-job-type":0} /tidb/cdc/default/default/task/position/03eb6b2c-343b-49c5-bc30-3b734541a901/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b" | grep id VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37062b7c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-dwwvr, pid:4236, start at 2024-05-06 19:54:40.779784251 +0800 CST m=+5.306626354 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:40.790 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:40.786 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:40.786 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37062cec000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-dwwvr, pid:4321, start at 2024-05-06 19:54:40.839364118 +0800 CST m=+5.315003776 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:40.846 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:40.827 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:40.827 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 19:54:43.872 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:54:43 CST 2024] <<<<<< run test case storage_csv_update success! >>>>>> [2024/05/06 19:54:44.073 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark7000] [2024/05/06 19:54:44.073 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark7000] check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37065128000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-fd5nv, pid:3512, start at 2024-05-06 19:54:43.17173169 +0800 CST m=+13.204948793 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:43.205 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:43.196 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:43.196 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37065128000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-fd5nv, pid:3512, start at 2024-05-06 19:54:43.17173169 +0800 CST m=+13.204948793 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:43.205 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:43.196 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:43.196 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5502.out cli changefeed resume -c test "cluster-id": "default" "id": "03eb6b2c-343b-49c5-bc30-3b734541a901", "cluster-id": "default" run task successfully wait process cdc.test exit for 2-th time... table mark.finish_mark_3 not exists for 15-th check, retry later [2024/05/06 19:54:45.086 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] [2024/05/06 19:54:45.179 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark5000] [ts=449576038228557829] [2024/05/06 19:54:45.179 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark5000] [2024/05/06 19:54:45.179 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576038228557829] [2024/05/06 19:54:45.262 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:45.265 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:54:45.269 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:45.275 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] wait process cdc.test exit for 3-th time... PASS coverage: 2.3% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:54:45 CST 2024] <<<<<< run test case clustered_index success! >>>>>> table batch_add_table.finish_mark not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... [2024/05/06 19:54:45.885 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365853647614662724, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449576038542868489, "checkpoint_time": "2024-05-06 19:54:44.920", "error": null }' + echo '{ "upstream_id": 7365853647614662724, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449576038542868489, "checkpoint_time": "2024-05-06 19:54:44.920", "error": null }' { "upstream_id": 7365853647614662724, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449576038542868489, "checkpoint_time": "2024-05-06 19:54:44.920", "error": null } ++ echo '{' '"upstream_id":' 7365853647614662724, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449576038542868489, '"checkpoint_time":' '"2024-05-06' '19:54:44.920",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365853647614662724, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449576038542868489, '"checkpoint_time":' '"2024-05-06' '19:54:44.920",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12992.out cli changefeed --changefeed-id custom-changefeed-name remove [2024/05/06 19:54:46.073 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37066c340005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-fd5nv, pid:3547, start at 2024-05-06 19:54:44.895301864 +0800 CST m=+14.803991171 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:44.923 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:44.929 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:44.929 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Mon May 6 19:54:45 CST 2024] <<<<<< START cdc server in processor_etcd_worker_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/orchestrator/ProcessorEtcdDelay=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_etcd_worker_delay.55645566.out server --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:46 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/79246d8d-eec7-4bd6-bc10-0ee6966b50aa {"id":"79246d8d-eec7-4bd6-bc10-0ee6966b50aa","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996483} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc154aa32 79246d8d-eec7-4bd6-bc10-0ee6966b50aa /tidb/cdc/default/default/upstream/7365853749452617429 {"id":7365853749452617429,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/79246d8d-eec7-4bd6-bc10-0ee6966b50aa {"id":"79246d8d-eec7-4bd6-bc10-0ee6966b50aa","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996483} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc154aa32 79246d8d-eec7-4bd6-bc10-0ee6966b50aa /tidb/cdc/default/default/upstream/7365853749452617429 {"id":7365853749452617429,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/79246d8d-eec7-4bd6-bc10-0ee6966b50aa {"id":"79246d8d-eec7-4bd6-bc10-0ee6966b50aa","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996483} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc154aa32 79246d8d-eec7-4bd6-bc10-0ee6966b50aa /tidb/cdc/default/default/upstream/7365853749452617429 {"id":7365853749452617429,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 449576038805012487 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/ PASS capture_id: 03eb6b2c-343b-49c5-bc30-3b734541a901 ***************** properties ***************** "workload"="core" "recordcount"="50000" "operationcount"="0" "requestdistribution"="uniform" "fieldcount"="3" "mysql.db"="consistent_replicate_nfs" "dotransactions"="false" "insertproportion"="0" "threadcount"="10" "mysql.port"="4000" "readproportion"="0" "scanproportion"="0" "updateproportion"="0" "mysql.host"="127.0.0.1" "mysql.user"="root" "readallfields"="true" ********************************************** Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } coverage: 2.2% of statements in github.com/pingcap/tiflow/... + set +x table mark.finish_mark_3 not exists for 16-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) table batch_update_to_no_batch.v1 exists table batch_update_to_no_batch.recover_and_insert not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370685840017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h, pid:10056, start at 2024-05-06 19:54:46.545694664 +0800 CST m=+5.367018425 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:46.555 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:46.547 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:46.547 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/http_api_tls_with_user_auth Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... [2024/05/06 19:54:47.274 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark8000] [2024/05/06 19:54:47.274 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark8000] [Pipeline] // cache go: downloading github.com/ardielle/ardielle-go v1.5.2 table batch_add_table.finish_mark exists check diff successfully [Pipeline] } Verifying downstream PD is started... [Pipeline] // dir [Pipeline] } wait process cdc.test exit for 1-th time... [Pipeline] // withCredentials [2024/05/06 19:54:47.960 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] [2024/05/06 19:54:47.976 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark6000] [ts=449576039172276229] [2024/05/06 19:54:47.976 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark6000] [2024/05/06 19:54:47.976 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576039172276229] [2024/05/06 19:54:47.978 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:47.981 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:54:48.060 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:48.065 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:54:48.159 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] [Pipeline] } + set +x [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Mon May 6 19:54:48 CST 2024] <<<<<< run test case batch_add_table success! >>>>>> [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Mon May 6 19:54:48 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.48074809.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo '' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:48 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a5ede318-6166-42a3-933f-126e25744c66 {"id":"a5ede318-6166-42a3-933f-126e25744c66","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996486} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1626c34 a5ede318-6166-42a3-933f-126e25744c66 /tidb/cdc/default/default/upstream/7365853760362197955 {"id":7365853760362197955,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a5ede318-6166-42a3-933f-126e25744c66 {"id":"a5ede318-6166-42a3-933f-126e25744c66","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996486} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1626c34 a5ede318-6166-42a3-933f-126e25744c66 /tidb/cdc/default/default/upstream/7365853760362197955 {"id":7365853760362197955,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a5ede318-6166-42a3-933f-126e25744c66 {"id":"a5ede318-6166-42a3-933f-126e25744c66","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996486} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1626c34 a5ede318-6166-42a3-933f-126e25744c66 /tidb/cdc/default/default/upstream/7365853760362197955 {"id":7365853760362197955,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Pipeline] // node table mark.finish_mark_3 not exists for 17-th check, retry later [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } 0 [Mon May 6 19:54:49 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.38203822.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server3.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server3 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // stage [Pipeline] } table batch_update_to_no_batch.recover_and_insert exists table batch_update_to_no_batch.finish_mark not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370685840017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h, pid:10056, start at 2024-05-06 19:54:46.545694664 +0800 CST m=+5.367018425 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:46.555 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:46.547 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:46.547 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3706870c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-hnk5h, pid:10142, start at 2024-05-06 19:54:46.634991557 +0800 CST m=+5.401769992 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:46.642 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:46.645 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:46.645 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... [2024/05/06 19:54:49.274 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 19:54:50.174 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] [2024/05/06 19:54:50.471 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark9000] [2024/05/06 19:54:50.471 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark9000] table mark.finish_mark_3 not exists for 18-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 19:54:51.359 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13079.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365853647614662724,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:54:51.805161705+08:00","start_ts":449576040312602629,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576040312602629,"checkpoint_ts":449576040312602629,"checkpoint_time":"2024-05-06 19:54:51.671"} PASS table batch_update_to_no_batch.finish_mark not exists for 2-th check, retry later ***************** properties ***************** "mysql.db"="changefeed_auto_stop_1" "scanproportion"="0" "mysql.port"="4000" "mysql.host"="127.0.0.1" "threadcount"="4" "operationcount"="0" "updateproportion"="0" "readallfields"="true" "readproportion"="0" "requestdistribution"="uniform" "dotransactions"="false" "workload"="core" "insertproportion"="0" "recordcount"="20" "mysql.user"="root" ********************************************** Run finished, takes 11.401843ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3477.0, Avg(us): 2170, Min(us): 1073, Max(us): 5892, 95th(us): 6000, 99th(us): 6000 coverage: 2.6% of statements in github.com/pingcap/tiflow/... ***************** properties ***************** "operationcount"="0" "requestdistribution"="uniform" "mysql.db"="changefeed_auto_stop_2" "threadcount"="4" "mysql.host"="127.0.0.1" "readallfields"="true" "mysql.user"="root" "dotransactions"="false" "readproportion"="0" "workload"="core" "updateproportion"="0" "recordcount"="20" "scanproportion"="0" "insertproportion"="0" "mysql.port"="4000" ********************************************** Run finished, takes 13.836193ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2022.5, Avg(us): 2606, Min(us): 1635, Max(us): 3900, 95th(us): 4000, 99th(us): 4000 Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:51 GMT < Content-Length: 871 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b5dff278-68e5-42b0-95d9-f397abcaec9e {"id":"b5dff278-68e5-42b0-95d9-f397abcaec9e","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996489} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc14e0371 b5dff278-68e5-42b0-95d9-f397abcaec9e /tidb/cdc/default/default/upstream/7365853737705987860 {"id":7365853737705987860,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b5dff278-68e5-42b0-95d9-f397abcaec9e {"id":"b5dff278-68e5-42b0-95d9-f397abcaec9e","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996489} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc14e0371 b5dff278-68e5-42b0-95d9-f397abcaec9e /tidb/cdc/default/default/upstream/7365853737705987860 {"id":7365853737705987860,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b5dff278-68e5-42b0-95d9-f397abcaec9e {"id":"b5dff278-68e5-42b0-95d9-f397abcaec9e","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996489} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc14e0371 b5dff278-68e5-42b0-95d9-f397abcaec9e /tidb/cdc/default/default/upstream/7365853737705987860 {"id":7365853737705987860,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ./t/bigint.test: ok! 138 test cases passed, take time 12.894885261 s time="2024-05-06T19:54:50+08:00" level=info msg="run test [bigint] ok" time="2024-05-06T19:54:50+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0001a61c0 {{} {} 0} {0 0} [0xc000116fc0] map[] 0 1 0xc0003005a0 false map[0xc000116fc0:map[0xc000116fc0:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" [2024/05/06 19:54:52.259 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] [2024/05/06 19:54:52.276 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark7000] [ts=449576040011137029] [2024/05/06 19:54:52.276 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark7000] [2024/05/06 19:54:52.276 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576040011137029] [2024/05/06 19:54:52.278 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:52.281 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:54:52.286 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:52.364 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] ***************** properties ***************** "dotransactions"="false" "recordcount"="20" "mysql.user"="root" "mysql.host"="127.0.0.1" "updateproportion"="0" "requestdistribution"="uniform" "operationcount"="0" "readproportion"="0" "insertproportion"="0" "readallfields"="true" "mysql.port"="4000" "mysql.db"="changefeed_auto_stop_3" "threadcount"="4" "workload"="core" "scanproportion"="0" ********************************************** Run finished, takes 11.514691ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3074.5, Avg(us): 2078, Min(us): 1032, Max(us): 4921, 95th(us): 5000, 99th(us): 5000 Create changefeed successfully! ID: e9333f03-c517-468c-adde-5c541749fed2 Info: {"upstream_id":7365853737705987860,"namespace":"default","id":"e9333f03-c517-468c-adde-5c541749fed2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:54:52.408687682+08:00","start_ts":449576040410906631,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576040410906631,"checkpoint_ts":449576040410906631,"checkpoint_time":"2024-05-06 19:54:52.046"} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:52 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0026d2bd0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576038618628101 MinTableBarrierTs:449576038880772101 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03eb6b2c-343b-49c5-bc30-3b734541a901 {"id":"03eb6b2c-343b-49c5-bc30-3b734541a901","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996480} /tidb/cdc/default/__cdc_meta__/capture/6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b {"id":"6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996474} /tidb/cdc/default/__cdc_meta__/capture/de31d40b-9586-4c2a-ba2b-8803a0f12710 {"id":"de31d40b-9586-4c2a-ba2b-8803a0f12710","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996489} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e662 6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e6ad 03eb6b2c-343b-49c5-bc30-3b734541a901 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e71a de31d40b-9586-4c2a-ba2b-8803a0f12710 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576038618628101,"min-table-barrier-ts":449576038880772101,"admin-job-type":0} /tidb/cdc/default/default/task/position/03eb6b2c-343b-49c5-bc30-3b734541a901/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/de31d40b-9586-4c2a-ba2b-8803a0f12710/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0026d2bd0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576038618628101 MinTableBarrierTs:449576038880772101 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03eb6b2c-343b-49c5-bc30-3b734541a901 {"id":"03eb6b2c-343b-49c5-bc30-3b734541a901","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996480} /tidb/cdc/default/__cdc_meta__/capture/6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b {"id":"6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996474} /tidb/cdc/default/__cdc_meta__/capture/de31d40b-9586-4c2a-ba2b-8803a0f12710 {"id":"de31d40b-9586-4c2a-ba2b-8803a0f12710","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996489} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e662 6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e6ad 03eb6b2c-343b-49c5-bc30-3b734541a901 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e71a de31d40b-9586-4c2a-ba2b-8803a0f12710 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576038618628101,"min-table-barrier-ts":449576038880772101,"admin-job-type":0} /tidb/cdc/default/default/task/position/03eb6b2c-343b-49c5-bc30-3b734541a901/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/de31d40b-9586-4c2a-ba2b-8803a0f12710/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0026d2bd0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576038618628101 MinTableBarrierTs:449576038880772101 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03eb6b2c-343b-49c5-bc30-3b734541a901 {"id":"03eb6b2c-343b-49c5-bc30-3b734541a901","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996480} /tidb/cdc/default/__cdc_meta__/capture/6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b {"id":"6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996474} /tidb/cdc/default/__cdc_meta__/capture/de31d40b-9586-4c2a-ba2b-8803a0f12710 {"id":"de31d40b-9586-4c2a-ba2b-8803a0f12710","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996489} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e662 6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e6ad 03eb6b2c-343b-49c5-bc30-3b734541a901 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e71a de31d40b-9586-4c2a-ba2b-8803a0f12710 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576038618628101,"min-table-barrier-ts":449576038880772101,"admin-job-type":0} /tidb/cdc/default/default/task/position/03eb6b2c-343b-49c5-bc30-3b734541a901/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6d47944c-ea6e-413b-bfb3-8d90fbbb9c0b/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/de31d40b-9586-4c2a-ba2b-8803a0f12710/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' ./t/composite_index.test: ok! 12 test cases passed, take time 1.5548806 s time="2024-05-06T19:54:52+08:00" level=info msg="run test [composite_index] ok" time="2024-05-06T19:54:52+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc00030c1b0 {{} {} 0} {0 0} [0xc000296000] map[] 0 1 0xc00003e840 false map[0xc000296000:map[0xc000296000:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" [2024/05/06 19:54:52.471 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 19-th check, retry later ***************** properties ***************** "operationcount"="0" "dotransactions"="false" "readallfields"="true" "mysql.port"="4000" "mysql.db"="changefeed_auto_stop_4" "mysql.host"="127.0.0.1" "threadcount"="4" "updateproportion"="0" "mysql.user"="root" "requestdistribution"="uniform" "recordcount"="20" "readproportion"="0" "workload"="core" "scanproportion"="0" "insertproportion"="0" ********************************************** Run finished, takes 12.742546ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2873.9, Avg(us): 2417, Min(us): 1237, Max(us): 5887, 95th(us): 6000, 99th(us): 6000 [Mon May 6 19:54:52 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1157011572.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 run task failed 1-th time, retry later + set +x split_and_random_merge scale: 20 [2024/05/06 19:54:53.375 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] [2024/05/06 19:54:53.570 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark10000] [2024/05/06 19:54:53.571 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark10000] table batch_update_to_no_batch.finish_mark not exists for 3-th check, retry later [2024/05/06 19:54:54.484 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] ./t/concurrent_ddl.test: ok! 8 test cases passed, take time 1.026473741 s time="2024-05-06T19:54:54+08:00" level=info msg="run test [concurrent_ddl] ok" time="2024-05-06T19:54:54+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0004980e0 {{} {} 0} {0 0} [0xc00062c1b0] map[] 0 1 0xc0004800c0 false map[0xc00062c1b0:map[0xc00062c1b0:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 20-th check, retry later run task failed 2-th time, retry later table processor_delay.t1 exists table processor_delay.t2 not exists for 1-th check, retry later [2024/05/06 19:54:55.386 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] [2024/05/06 19:54:55.470 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark8000] [ts=449576040849997829] [2024/05/06 19:54:55.470 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark8000] [2024/05/06 19:54:55.470 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576040849997829] [2024/05/06 19:54:55.472 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:55.476 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:54:55.480 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:55.485 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:54:55.571 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] table batch_update_to_no_batch.finish_mark not exists for 4-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:55 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48d2d831-7b63-4bd7-8780-b4a9fb2330a2 {"id":"48d2d831-7b63-4bd7-8780-b4a9fb2330a2","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996492} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc17a7e84 48d2d831-7b63-4bd7-8780-b4a9fb2330a2 /tidb/cdc/default/default/upstream/7365853782132263093 {"id":7365853782132263093,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48d2d831-7b63-4bd7-8780-b4a9fb2330a2 {"id":"48d2d831-7b63-4bd7-8780-b4a9fb2330a2","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996492} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc17a7e84 48d2d831-7b63-4bd7-8780-b4a9fb2330a2 /tidb/cdc/default/default/upstream/7365853782132263093 {"id":7365853782132263093,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48d2d831-7b63-4bd7-8780-b4a9fb2330a2 {"id":"48d2d831-7b63-4bd7-8780-b4a9fb2330a2","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996492} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc17a7e84 48d2d831-7b63-4bd7-8780-b4a9fb2330a2 /tidb/cdc/default/default/upstream/7365853782132263093 {"id":7365853782132263093,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:54:55 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedError=1*return(true);github.com/pingcap/tiflow/cdc/processor/ProcessorUpdatePositionDelaying=sleep(1000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1162811630.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:54:56.595 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark11000] [2024/05/06 19:54:56.595 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark11000] + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365853647614662724, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449576041439559682, "checkpoint_time": "2024-05-06 19:54:55.970", "error": null }' + echo '{ "upstream_id": 7365853647614662724, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449576041439559682, "checkpoint_time": "2024-05-06 19:54:55.970", "error": null }' { "upstream_id": 7365853647614662724, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449576041439559682, "checkpoint_time": "2024-05-06 19:54:55.970", "error": null } ++ echo '{' '"upstream_id":' 7365853647614662724, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449576041439559682, '"checkpoint_time":' '"2024-05-06' '19:54:55.970",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365853647614662724, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449576041439559682, '"checkpoint_time":' '"2024-05-06' '19:54:55.970",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] INSERT - Takes(s): 10.0, Count: 25060, OPS: 2507.2, Avg(us): 3957, Min(us): 729, Max(us): 95414, 95th(us): 4000, 99th(us): 85000 [2024/05/06 19:54:56.661 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 21-th check, retry later table processor_delay.t2 exists table processor_delay.t3 not exists for 1-th check, retry later table batch_update_to_no_batch.finish_mark not exists for 5-th check, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13203.out cli unsafe delete-service-gc-safepoint Confirm that you know what this command will do and use it at your own risk [Y/N] [2024/05/06 19:54:57.659 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] CDC service GC safepoint truncated in PD! PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370730100004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-mzzb9, pid:6796, start at 2024-05-06 19:54:57.415801291 +0800 CST m=+5.277286200 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:57.423 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:57.412 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:57.412 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_rocks/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 19:54:58.659 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] [2024/05/06 19:54:58.674 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] [2024/05/06 19:54:58.689 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark9000] [ts=449576041672605701] [2024/05/06 19:54:58.689 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark9000] [2024/05/06 19:54:58.689 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576041672605701] [2024/05/06 19:54:58.691 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:58.760 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:54:58.765 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:54:58.770 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 22-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... table processor_delay.t3 exists table processor_delay.t4 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:54:58 GMT < Content-Length: 1295 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e943387-59a7-4606-aad6-45ec93c41afc {"id":"3e943387-59a7-4606-aad6-45ec93c41afc","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996496} /tidb/cdc/default/__cdc_meta__/capture/48d2d831-7b63-4bd7-8780-b4a9fb2330a2 {"id":"48d2d831-7b63-4bd7-8780-b4a9fb2330a2","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996492} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc17a7e84 48d2d831-7b63-4bd7-8780-b4a9fb2330a2 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc17a7eb8 3e943387-59a7-4606-aad6-45ec93c41afc /tidb/cdc/default/default/upstream/7365853782132263093 {"id":7365853782132263093,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e943387-59a7-4606-aad6-45ec93c41afc {"id":"3e943387-59a7-4606-aad6-45ec93c41afc","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996496} /tidb/cdc/default/__cdc_meta__/capture/48d2d831-7b63-4bd7-8780-b4a9fb2330a2 {"id":"48d2d831-7b63-4bd7-8780-b4a9fb2330a2","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996492} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc17a7e84 48d2d831-7b63-4bd7-8780-b4a9fb2330a2 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc17a7eb8 3e943387-59a7-4606-aad6-45ec93c41afc /tidb/cdc/default/default/upstream/7365853782132263093 {"id":7365853782132263093,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e943387-59a7-4606-aad6-45ec93c41afc {"id":"3e943387-59a7-4606-aad6-45ec93c41afc","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996496} /tidb/cdc/default/__cdc_meta__/capture/48d2d831-7b63-4bd7-8780-b4a9fb2330a2 {"id":"48d2d831-7b63-4bd7-8780-b4a9fb2330a2","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996492} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc17a7e84 48d2d831-7b63-4bd7-8780-b4a9fb2330a2 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc17a7eb8 3e943387-59a7-4606-aad6-45ec93c41afc /tidb/cdc/default/default/upstream/7365853782132263093 {"id":7365853782132263093,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 46a6f715-e78b-4deb-86e5-685403fda962 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=46a6f715-e78b-4deb-86e5-685403fda962 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 46a6f715-e78b-4deb-86e5-685403fda962 -s + info='{ "upstream_id": 7365853782132263093, "namespace": "default", "id": "46a6f715-e78b-4deb-86e5-685403fda962", "state": "normal", "checkpoint_tso": 449576040293203969, "checkpoint_time": "2024-05-06 19:54:51.597", "error": null }' + echo '{ "upstream_id": 7365853782132263093, "namespace": "default", "id": "46a6f715-e78b-4deb-86e5-685403fda962", "state": "normal", "checkpoint_tso": 449576040293203969, "checkpoint_time": "2024-05-06 19:54:51.597", "error": null }' { "upstream_id": 7365853782132263093, "namespace": "default", "id": "46a6f715-e78b-4deb-86e5-685403fda962", "state": "normal", "checkpoint_tso": 449576040293203969, "checkpoint_time": "2024-05-06 19:54:51.597", "error": null } ++ echo '{' '"upstream_id":' 7365853782132263093, '"namespace":' '"default",' '"id":' '"46a6f715-e78b-4deb-86e5-685403fda962",' '"state":' '"normal",' '"checkpoint_tso":' 449576040293203969, '"checkpoint_time":' '"2024-05-06' '19:54:51.597",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365853782132263093, '"namespace":' '"default",' '"id":' '"46a6f715-e78b-4deb-86e5-685403fda962",' '"state":' '"normal",' '"checkpoint_tso":' 449576040293203969, '"checkpoint_time":' '"2024-05-06' '19:54:51.597",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table changefeed_auto_stop_1.usertable not exists for 1-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13236.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' reset and all metadata truncated in PD! PASS table batch_update_to_no_batch.finish_mark exists [2024/05/06 19:54:59.573 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark12000] [2024/05/06 19:54:59.573 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark12000] [2024/05/06 19:54:59.671 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] check diff successfully coverage: 2.2% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370730100004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-mzzb9, pid:6796, start at 2024-05-06 19:54:57.415801291 +0800 CST m=+5.277286200 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:57.423 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:57.412 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:57.412 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370730d40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-mzzb9, pid:6882, start at 2024-05-06 19:54:57.496236188 +0800 CST m=+5.302753900 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:56:57.503 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:54:57.461 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:44:57.461 +0800 All versions after safe point can be accessed. (DO NOT EDIT) wait process cdc.test exit for 2-th time... Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_source/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_source/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } "is-owner": true, run task successfully nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 23-th check, retry later [2024/05/06 19:55:00.674 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] start tidb cluster in /tmp/tidb_cdc_test/multi_rocks Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... ./t/date_formats.test: ok! 99 test cases passed, take time 5.297216394 s time="2024-05-06T19:54:59+08:00" level=info msg="run test [date_formats] ok" time="2024-05-06T19:54:59+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0001a61b8 {{} {} 0} {0 0} [0xc000138f30] map[] 0 1 0xc000622ba0 false map[0xc000138f30:map[0xc000138f30:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" + set +x wait process cdc.test exit for 3-th time... [2024/05/06 19:55:01.573 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] [2024/05/06 19:55:01.684 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] table processor_delay.t4 exists table processor_delay.t5 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:55:01 CST 2024] <<<<<< run test case batch_update_to_no_batch success! >>>>>> table changefeed_auto_stop_1.usertable exists table changefeed_auto_stop_2.usertable not exists for 1-th check, retry later [2024/05/06 19:55:01.769 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark10000] [ts=449576042554195977] [2024/05/06 19:55:01.769 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark10000] [2024/05/06 19:55:01.769 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576042554195977] [2024/05/06 19:55:01.771 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:01.860 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:55:01.866 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:01.871 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8209.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_3 not exists for 24-th check, retry later [2024/05/06 19:55:02.771 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] [2024/05/06 19:55:02.893 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark13000] [2024/05/06 19:55:02.893 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark13000] table processor_delay.t5 exists table processor_delay.t6 not exists for 1-th check, retry later table changefeed_auto_stop_2.usertable exists table changefeed_auto_stop_3.usertable not exists for 1-th check, retry later [2024/05/06 19:55:03.671 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] Starting Upstream TiDB... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13344.out cli unsafe resolve-lock --region=18 Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449576043272208385 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576043272208385 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:55:04 CST 2024] <<<<<< START cdc server in multi_source case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.82488250.out server --log-file /tmp/tidb_cdc_test/multi_source/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_source/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:55:04.787 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] [2024/05/06 19:55:04.893 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] [2024/05/06 19:55:04.983 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark11000] [ts=449576043402756101] [2024/05/06 19:55:04.983 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark11000] [2024/05/06 19:55:04.983 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576043402756101] [2024/05/06 19:55:05.060 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:05.067 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:55:05.071 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:05.076 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_3 not exists for 25-th check, retry later table processor_delay.t6 exists table processor_delay.t7 not exists for 1-th check, retry later table changefeed_auto_stop_3.usertable not exists for 2-th check, retry later [2024/05/06 19:55:05.208 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/06 19:55:05.210 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/06 19:55:05.211 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=671896cf-401e-4092-bd9a-9160eb98eddc] [2024/05/06 19:55:05.211 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/06 19:55:05.211 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/06 19:55:05.384 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=dc0341b5-a12c-42df-a8ff-2e511d84a438] [captureID=2df0df91-4b4f-4f75-8268-545024bdcf73] [processorDetail="{\"table_ids\":[]}"] [2024/05/06 19:55:05.584 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=dc0341b5-a12c-42df-a8ff-2e511d84a438] [captureID=671896cf-401e-4092-bd9a-9160eb98eddc] [processorDetail="{\"table_ids\":[108]}"] [2024/05/06 19:55:05.683 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] [2024/05/06 19:55:05.784 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=dc0341b5-a12c-42df-a8ff-2e511d84a438] [captureID=afbc0a4a-139f-4d5d-a458-05d51fbedd73] [processorDetail="{\"table_ids\":[106]}"] [2024/05/06 19:55:05.785 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"2df0df91-4b4f-4f75-8268-545024bdcf73\":[],\"671896cf-401e-4092-bd9a-9160eb98eddc\":[{\"ID\":108,\"Changefeed\":\"dc0341b5-a12c-42df-a8ff-2e511d84a438\"}],\"afbc0a4a-139f-4d5d-a458-05d51fbedd73\":[{\"ID\":106,\"Changefeed\":\"dc0341b5-a12c-42df-a8ff-2e511d84a438\"}]}"] [2024/05/06 19:55:05.785 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=dc0341b5-a12c-42df-a8ff-2e511d84a438&target-cp-id=afbc0a4a-139f-4d5d-a458-05d51fbedd73&table-id=108"] [2024/05/06 19:55:05.835 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=108] [2024/05/06 19:55:05.835 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=671896cf-401e-4092-bd9a-9160eb98eddc] [targetCapture=afbc0a4a-139f-4d5d-a458-05d51fbedd73] table move_table.check1 exists check diff successfully Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:55:06.177 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark14000] [2024/05/06 19:55:06.177 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark14000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13377.out cli unsafe resolve-lock --region=18 --ts=449576042789601282 PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... INSERT - Takes(s): 20.0, Count: 36255, OPS: 1813.2, Avg(us): 5495, Min(us): 729, Max(us): 95414, 95th(us): 47000, 99th(us): 88000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) empty select id, val from test.availability1 where id=1 run task successfully test_owner_cleanup_stale_tasks pass table mark.finish_mark_3 not exists for 26-th check, retry later Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used [2024/05/06 19:55:07.069 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] wait process cdc.test exit for 1-th time... table processor_delay.t7 exists table processor_delay.t8 not exists for 1-th check, retry later table changefeed_auto_stop_3.usertable exists table changefeed_auto_stop_4.usertable not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:55:07 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/050a0076-15eb-4648-9f2d-5531d5bc2e20 {"id":"050a0076-15eb-4648-9f2d-5531d5bc2e20","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996504} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc19f7541 050a0076-15eb-4648-9f2d-5531d5bc2e20 /tidb/cdc/default/default/upstream/7365853823033520764 {"id":7365853823033520764,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/050a0076-15eb-4648-9f2d-5531d5bc2e20 {"id":"050a0076-15eb-4648-9f2d-5531d5bc2e20","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996504} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc19f7541 050a0076-15eb-4648-9f2d-5531d5bc2e20 /tidb/cdc/default/default/upstream/7365853823033520764 {"id":7365853823033520764,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/050a0076-15eb-4648-9f2d-5531d5bc2e20 {"id":"050a0076-15eb-4648-9f2d-5531d5bc2e20","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996504} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc19f7541 050a0076-15eb-4648-9f2d-5531d5bc2e20 /tidb/cdc/default/default/upstream/7365853823033520764 {"id":7365853823033520764,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8307.out cli changefeed create --start-ts=449576043272208385 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ [2024/05/06 19:55:07.764 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] [2024/05/06 19:55:07.866 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark12000] [ts=449576044241616900] [2024/05/06 19:55:07.866 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark12000] [2024/05/06 19:55:07.866 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576044241616900] [2024/05/06 19:55:07.869 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 167k 4850 --:--:-- --:--:-- --:--:-- 207k { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" wait process cdc.test exit for 2-th time... Create changefeed successfully! ID: 9f0e59ba-901f-4221-b166-57e754bdb049 Info: {"upstream_id":7365853823033520764,"namespace":"default","id":"9f0e59ba-901f-4221-b166-57e754bdb049","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:55:07.95023809+08:00","start_ts":449576043272208385,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576043272208385,"checkpoint_ts":449576043272208385,"checkpoint_time":"2024-05-06 19:55:02.961"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:55:07.959 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:55:07.964 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:07.971 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:55:08.177 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3707c60c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:35102, start at 2024-05-06 19:55:07.033577232 +0800 CST m=+14.449311399 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:07.041 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:07.011 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:07.011 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3707c60c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:35102, start at 2024-05-06 19:55:07.033577232 +0800 CST m=+14.449311399 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:07.041 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:07.011 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:07.011 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3707c5140003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:35188, start at 2024-05-06 19:55:06.951132752 +0800 CST m=+14.302284194 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:06.958 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:06.949 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:06.949 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table mark.finish_mark_3 not exists for 27-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit run test case test_owner_retryable_error [Mon May 6 19:55:08 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/capture/capture-campaign-compacted-error=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.41014103.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ./t/drop.test: ok! 48 test cases passed, take time 8.709460346 s time="2024-05-06T19:55:09+08:00" level=info msg="run test [drop] ok" time="2024-05-06T19:55:09+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0000da2e8 {{} {} 0} {0 0} [0xc000139170] map[] 0 1 0xc000480ea0 false map[0xc000139170:map[0xc000139170:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" [2024/05/06 19:55:09.161 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] table processor_delay.t8 exists table processor_delay.t9 not exists for 1-th check, retry later + set +x go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f table changefeed_auto_stop_4.usertable exists check diff failed 1-th time, retry later [2024/05/06 19:55:09.471 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark15000] [2024/05/06 19:55:09.471 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark15000] go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 [2024/05/06 19:55:03.837 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 2 another is 4"] go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda [2024/05/06 19:55:10.296 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/06 19:55:10.299 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/06 19:55:10.299 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=671896cf-401e-4092-bd9a-9160eb98eddc] [2024/05/06 19:55:10.300 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/06 19:55:10.300 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] go: downloading golang.org/x/text v0.14.0 [2024/05/06 19:55:10.262 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] [2024/05/06 19:55:10.485 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=dc0341b5-a12c-42df-a8ff-2e511d84a438] [captureID=2df0df91-4b4f-4f75-8268-545024bdcf73] [processorDetail="{\"table_ids\":[]}"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3707eeb40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-89kh8, pid:4449, start at 2024-05-06 19:55:09.638445849 +0800 CST m=+5.344709764 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:09.647 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:09.613 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:09.613 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:55:10.684 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=dc0341b5-a12c-42df-a8ff-2e511d84a438] [captureID=671896cf-401e-4092-bd9a-9160eb98eddc] [processorDetail="{\"table_ids\":[110]}"] [2024/05/06 19:55:10.885 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=dc0341b5-a12c-42df-a8ff-2e511d84a438] [captureID=afbc0a4a-139f-4d5d-a458-05d51fbedd73] [processorDetail="{\"table_ids\":[108]}"] [2024/05/06 19:55:10.885 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"2df0df91-4b4f-4f75-8268-545024bdcf73\":[],\"671896cf-401e-4092-bd9a-9160eb98eddc\":[{\"ID\":110,\"Changefeed\":\"dc0341b5-a12c-42df-a8ff-2e511d84a438\"}],\"afbc0a4a-139f-4d5d-a458-05d51fbedd73\":[{\"ID\":108,\"Changefeed\":\"dc0341b5-a12c-42df-a8ff-2e511d84a438\"}]}"] [2024/05/06 19:55:10.885 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=dc0341b5-a12c-42df-a8ff-2e511d84a438&target-cp-id=2df0df91-4b4f-4f75-8268-545024bdcf73&table-id=110"] table mark.finish_mark_3 not exists for 28-th check, retry later ./t/mysql_replace.test: ok! 16 test cases passed, take time 1.048330019 s time="2024-05-06T19:55:10+08:00" level=info msg="run test [mysql_replace] ok" Great, All tests passed mysqltest end [2024/05/06 19:55:10.934 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=110] [2024/05/06 19:55:10.934 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=671896cf-401e-4092-bd9a-9160eb98eddc] [targetCapture=2df0df91-4b4f-4f75-8268-545024bdcf73] check diff successfully } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 200k 0 --:--:-- --:--:-- --:--:-- 177k { "version": "v8.1.0-alpha-23-ge1a27adc6", "git_hash": "e1a27adc69acc9d50db0c3fea2945bbd6d983e30", "id": "808dde80-1bf4-4597-88ab-398efb499d33", "pid": 12310, "is_owner": true table test.finish_mark not exists for 1-th check, retry later [2024/05/06 19:55:11.259 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] [2024/05/06 19:55:11.363 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark13000] [ts=449576045175373842] [2024/05/06 19:55:11.363 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark13000] [2024/05/06 19:55:11.363 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576045175373842] [2024/05/06 19:55:11.365 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:11.369 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:55:11.375 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] }wait process cdc.test exit for 1-th time... The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls_with_user_auth Starting TLS PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 table move_table.check2 not exists for 1-th check, retry later [2024/05/06 19:55:11.462 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:55:11.471 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_resume_with_checkpoint_ts/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table processor_delay.t9 exists table processor_delay.t10 not exists for 1-th check, retry later check diff successfully wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/google/btree v1.1.2 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading cloud.google.com/go v0.112.2 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/spkg/bom v1.0.0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:55:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00147e7e0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576044123652101 MinTableBarrierTs:449576045434372104 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576044123652101, checkpointTs: 449576044123652101, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576044123652101, checkpointTs: 449576044123652101, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576044123652101, checkpointTs: 449576044123652101, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/08b8a63c-85cd-4891-80c4-5a40e2438d55 {"id":"08b8a63c-85cd-4891-80c4-5a40e2438d55","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996509} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e7d7 08b8a63c-85cd-4891-80c4-5a40e2438d55 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576044123652101,"min-table-barrier-ts":449576045434372104,"admin-job-type":0} /tidb/cdc/default/default/task/position/08b8a63c-85cd-4891-80c4-5a40e2438d55/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00147e7e0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576044123652101 MinTableBarrierTs:449576045434372104 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576044123652101, checkpointTs: 449576044123652101, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576044123652101, checkpointTs: 449576044123652101, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576044123652101, checkpointTs: 449576044123652101, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/08b8a63c-85cd-4891-80c4-5a40e2438d55 {"id":"08b8a63c-85cd-4891-80c4-5a40e2438d55","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996509} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e7d7 08b8a63c-85cd-4891-80c4-5a40e2438d55 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576044123652101,"min-table-barrier-ts":449576045434372104,"admin-job-type":0} /tidb/cdc/default/default/task/position/08b8a63c-85cd-4891-80c4-5a40e2438d55/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00147e7e0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576044123652101 MinTableBarrierTs:449576045434372104 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576044123652101, checkpointTs: 449576044123652101, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576044123652101, checkpointTs: 449576044123652101, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576044123652101, checkpointTs: 449576044123652101, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/08b8a63c-85cd-4891-80c4-5a40e2438d55 {"id":"08b8a63c-85cd-4891-80c4-5a40e2438d55","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996509} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e7d7 08b8a63c-85cd-4891-80c4-5a40e2438d55 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576044123652101,"min-table-barrier-ts":449576045434372104,"admin-job-type":0} /tidb/cdc/default/default/task/position/08b8a63c-85cd-4891-80c4-5a40e2438d55/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' Starting TLS TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading cloud.google.com/go/iam v1.1.7 wait process cdc.test exit for 3-th time... [2024/05/06 19:55:12.275 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] [2024/05/06 19:55:12.373 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark16000] [2024/05/06 19:55:12.373 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark16000] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3707eeb40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-89kh8, pid:4449, start at 2024-05-06 19:55:09.638445849 +0800 CST m=+5.344709764 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:09.647 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:09.613 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:09.613 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3707ef540007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-89kh8, pid:4501, start at 2024-05-06 19:55:09.659004188 +0800 CST m=+5.312196670 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:09.668 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:09.653 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:09.653 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 2-th time... go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 table mark.finish_mark_3 not exists for 29-th check, retry later go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:55:13 CST 2024] <<<<<< run test case cli_with_auth success! >>>>>> table test.finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 3-th time... go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:55:13 CST 2024] <<<<<< run test case changefeed_auto_stop success! >>>>>> go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 "is-owner": true, run task successfully table processor_delay.t10 exists table processor_delay.t11 not exists for 1-th check, retry later [2024/05/06 19:55:13.569 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] table move_table.check2 exists check diff successfully Starting TLS TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... [2024/05/06 19:55:14.359 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] [2024/05/06 19:55:14.373 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] [2024/05/06 19:55:14.461 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark14000] [ts=449576046024196101] [2024/05/06 19:55:14.461 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark14000] [2024/05/06 19:55:14.461 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576046024196101] [2024/05/06 19:55:14.464 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:14.468 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:55:14.559 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 30-th check, retry later [2024/05/06 19:55:14.568 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... owner pid: 4106 owner id 08b8a63c-85cd-4891-80c4-5a40e2438d55 [Mon May 6 19:55:14 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/owner-run-with-error=1*return(true);github.com/pingcap/tiflow/cdc/capture/capture-resign-failed=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.42334235.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 19:55:14 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=return(true);github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.57495751.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 3-th time... table processor_delay.t11 exists table processor_delay.t12 not exists for 1-th check, retry later [2024/05/06 19:55:15.583 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] table test.finish_mark not exists for 3-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:55:15 CST 2024] <<<<<< run test case move_table success! >>>>>> [2024/05/06 19:55:15.764 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark17000] [2024/05/06 19:55:15.764 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark17000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:55:16.562 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 Verifying downstream PD is started... table mark.finish_mark_3 not exists for 31-th check, retry later table processor_delay.t12 exists table processor_delay.t13 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:55:17 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/512444c1-549e-43f6-8df5-a6a5f1b78368 {"id":"512444c1-549e-43f6-8df5-a6a5f1b78368","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996514} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1d11533 512444c1-549e-43f6-8df5-a6a5f1b78368 /tidb/cdc/default/default/upstream/7365853887776047536 {"id":7365853887776047536,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/512444c1-549e-43f6-8df5-a6a5f1b78368 {"id":"512444c1-549e-43f6-8df5-a6a5f1b78368","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996514} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1d11533 512444c1-549e-43f6-8df5-a6a5f1b78368 /tidb/cdc/default/default/upstream/7365853887776047536 {"id":7365853887776047536,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/512444c1-549e-43f6-8df5-a6a5f1b78368 {"id":"512444c1-549e-43f6-8df5-a6a5f1b78368","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996514} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1d11533 512444c1-549e-43f6-8df5-a6a5f1b78368 /tidb/cdc/default/default/upstream/7365853887776047536 {"id":7365853887776047536,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.cli.5794.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 table test.finish_mark not exists for 4-th check, retry later go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/reflect2 v1.0.2 [2024/05/06 19:55:17.667 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] [2024/05/06 19:55:17.764 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] [2024/05/06 19:55:17.862 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark15000] [ts=449576046863056901] [2024/05/06 19:55:17.862 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark15000] [2024/05/06 19:55:17.862 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576046863056901] [2024/05/06 19:55:17.866 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:17.870 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd [2024/05/06 19:55:17.962 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:17.971 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Create changefeed successfully! ID: 66d7a011-45b1-479f-a14d-a97714ad2a87 Info: {"upstream_id":7365853887776047536,"namespace":"default","id":"66d7a011-45b1-479f-a14d-a97714ad2a87","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:55:18.127951184+08:00","start_ts":449576047217999875,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576047217999875,"checkpoint_ts":449576047217999875,"checkpoint_time":"2024-05-06 19:55:18.013"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:55:18 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0012a8fc0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576047020081153 MinTableBarrierTs:449576047020081153 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/08b8a63c-85cd-4891-80c4-5a40e2438d55 {"id":"08b8a63c-85cd-4891-80c4-5a40e2438d55","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996509} /tidb/cdc/default/__cdc_meta__/capture/cce4cf7d-c514-4f07-8568-b0c57f951bf6 {"id":"cce4cf7d-c514-4f07-8568-b0c57f951bf6","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996515} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e7d7 08b8a63c-85cd-4891-80c4-5a40e2438d55 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e825 cce4cf7d-c514-4f07-8568-b0c57f951bf6 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576047020081153,"min-table-barrier-ts":449576047020081153,"admin-job-type":0} /tidb/cdc/default/default/task/position/08b8a63c-85cd-4891-80c4-5a40e2438d55/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cce4cf7d-c514-4f07-8568-b0c57f951bf6/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0012a8fc0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576047020081153 MinTableBarrierTs:449576047020081153 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/08b8a63c-85cd-4891-80c4-5a40e2438d55 {"id":"08b8a63c-85cd-4891-80c4-5a40e2438d55","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996509} /tidb/cdc/default/__cdc_meta__/capture/cce4cf7d-c514-4f07-8568-b0c57f951bf6 {"id":"cce4cf7d-c514-4f07-8568-b0c57f951bf6","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996515} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e7d7 08b8a63c-85cd-4891-80c4-5a40e2438d55 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e825 cce4cf7d-c514-4f07-8568-b0c57f951bf6 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576047020081153,"min-table-barrier-ts":449576047020081153,"admin-job-type":0} /tidb/cdc/default/default/task/position/08b8a63c-85cd-4891-80c4-5a40e2438d55/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cce4cf7d-c514-4f07-8568-b0c57f951bf6/15d744e6-9542-4a9e-820f-73b6ebaa1984 + grep -q 'failed to get info:' {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0012a8fc0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576047020081153 MinTableBarrierTs:449576047020081153 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/08b8a63c-85cd-4891-80c4-5a40e2438d55 {"id":"08b8a63c-85cd-4891-80c4-5a40e2438d55","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996509} /tidb/cdc/default/__cdc_meta__/capture/cce4cf7d-c514-4f07-8568-b0c57f951bf6 {"id":"cce4cf7d-c514-4f07-8568-b0c57f951bf6","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996515} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e7d7 08b8a63c-85cd-4891-80c4-5a40e2438d55 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e825 cce4cf7d-c514-4f07-8568-b0c57f951bf6 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576047020081153,"min-table-barrier-ts":449576047020081153,"admin-job-type":0} /tidb/cdc/default/default/task/position/08b8a63c-85cd-4891-80c4-5a40e2438d55/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cce4cf7d-c514-4f07-8568-b0c57f951bf6/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "08b8a63c-85cd-4891-80c4-5a40e2438d55" | grep id ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:55:18.575 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] INSERT - Takes(s): 30.0, Count: 49380, OPS: 1646.3, Avg(us): 6057, Min(us): 729, Max(us): 95414, 95th(us): 72000, 99th(us): 88000 Run finished, takes 30.403815929s INSERT - Takes(s): 30.4, Count: 49999, OPS: 1644.8, Avg(us): 6046, Min(us): 729, Max(us): 95414, 95th(us): 71000, 99th(us): 88000 table consistent_replicate_nfs.usertable exists wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... table mark.finish_mark_3 not exists for 32-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... [2024/05/06 19:55:19.071 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark18000] [2024/05/06 19:55:19.071 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark18000] Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_delay.t13 exists table processor_delay.t14 not exists for 1-th check, retry later process cdc.test already exit + set +x table test.finish_mark not exists for 5-th check, retry later "cluster-id": "default" "id": "cce4cf7d-c514-4f07-8568-b0c57f951bf6", "cluster-id": "default" run task successfully cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' [2024/05/06 19:55:19.862 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] 1 run task successfully table processor_resolved_ts_fallback.t1 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37088cf00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:36754, start at 2024-05-06 19:55:19.77432883 +0800 CST m=+5.443467070 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:19.781 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:19.740 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:19.740 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Mon May 6 19:55:20 CST 2024] <<<<<< START cdc server in http_api_tls_with_user_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates ']' + GO_FAILPOINTS= + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls_with_user_auth.3681736819.out server --log-file /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_http_api_tls_with_user_auth_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_data_http_api_tls_with_user_auth_tls1 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls_with_user_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info [2024/05/06 19:55:20.591 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] [2024/05/06 19:55:20.605 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark16000] [ts=449576047832989701] [2024/05/06 19:55:20.606 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark16000] [2024/05/06 19:55:20.606 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576047832989701] [2024/05/06 19:55:20.608 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:20.659 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:55:20.664 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:20.670 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 33-th check, retry later capture_id: cce4cf7d-c514-4f07-8568-b0c57f951bf6 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 35 100 35 0 0 26119 0 --:--:-- --:--:-- --:--:-- 35000 { "status": true, "message": "" }ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '08b8a63c-85cd-4891-80c4-5a40e2438d55' 08b8a63c-85cd-4891-80c4-5a40e2438d55 run task successfully ps -C cdc.test -o pid= | awk '{print $1}' | wc -l | grep 2 2 run task successfully test_owner_retryable_error pass Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:55:21.072 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] table processor_delay.t14 exists table test.finish_mark not exists for 6-th check, retry later table processor_delay.t15 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... [2024/05/06 19:55:21.876 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] table processor_resolved_ts_fallback.t1 exists [Mon May 6 19:55:21 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.58865888.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:55:21.973 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark19000] [2024/05/06 19:55:21.973 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark19000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... table mark.finish_mark_3 not exists for 34-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit run test case test_gap_between_watch_capture [Mon May 6 19:55:23 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.44024404.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 7-th check, retry later table processor_delay.t15 exists table processor_delay.t16 not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [2024/05/06 19:55:23.162 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [2024/05/06 19:55:23.962 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] [2024/05/06 19:55:23.973 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark19000] [2024/05/06 19:55:23.977 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark17000] [ts=449576048750493701] [2024/05/06 19:55:23.977 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark17000] [2024/05/06 19:55:23.977 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576048750493701] [2024/05/06 19:55:23.979 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:23.982 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:55:23.987 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:23.992 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:55:23 GMT < Content-Length: 1281 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/531822ee-e263-4974-9812-add93d8c7a1c {"id":"531822ee-e263-4974-9812-add93d8c7a1c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996521} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dc2016631 531822ee-e263-4974-9812-add93d8c7a1c /tidb/cdc/default/default/upstream/7365853933390517639 {"id":7365853933390517639,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/531822ee-e263-4974-9812-add93d8c7a1c {"id":"531822ee-e263-4974-9812-add93d8c7a1c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996521} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dc2016631 531822ee-e263-4974-9812-add93d8c7a1c /tidb/cdc/default/default/upstream/7365853933390517639 {"id":7365853933390517639,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/531822ee-e263-4974-9812-add93d8c7a1c {"id":"531822ee-e263-4974-9812-add93d8c7a1c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996521} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dc2016631 531822ee-e263-4974-9812-add93d8c7a1c /tidb/cdc/default/default/upstream/7365853933390517639 {"id":7365853933390517639,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x [Pipeline] // container [Pipeline] } [2024/05/06 19:55:24.370 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20001] [2024/05/06 19:55:24.370 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] [2024/05/06 19:55:24.372 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20002] [2024/05/06 19:55:24.372 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] [2024/05/06 19:55:24.374 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20003] [2024/05/06 19:55:24.374 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] [2024/05/06 19:55:24.374 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20004] [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [2024/05/06 19:55:24.572 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark20000] [2024/05/06 19:55:24.572 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20004] [Pipeline] // podTemplate ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } table mark.finish_mark_3 not exists for 35-th check, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:55:25.177 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:55:24 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/66d7a011-45b1-479f-a14d-a97714ad2a87 {UpstreamID:7365853887776047536 Namespace:default ID:66d7a011-45b1-479f-a14d-a97714ad2a87 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:55:18.127951184 +0800 CST StartTs:449576047217999875 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003170630 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576047243952131} {CheckpointTs:449576048882614285 MinTableBarrierTs:449576048882614285 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/512444c1-549e-43f6-8df5-a6a5f1b78368 {"id":"512444c1-549e-43f6-8df5-a6a5f1b78368","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996514} /tidb/cdc/default/__cdc_meta__/capture/b13b93ac-ab2c-48a6-93ec-320e944238e8 {"id":"b13b93ac-ab2c-48a6-93ec-320e944238e8","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996522} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1d11533 512444c1-549e-43f6-8df5-a6a5f1b78368 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1d115a7 b13b93ac-ab2c-48a6-93ec-320e944238e8 /tidb/cdc/default/default/changefeed/info/66d7a011-45b1-479f-a14d-a97714ad2a87 {"upstream-id":7365853887776047536,"namespace":"default","changefeed-id":"66d7a011-45b1-479f-a14d-a97714ad2a87","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:55:18.127951184+08:00","start-ts":449576047217999875,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576047243952131} /tidb/cdc/default/default/changefeed/status/66d7a011-45b1-479f-a14d-a97714ad2a87 {"checkpoint-ts":449576048882614285,"min-table-barrier-ts":449576048882614285,"admin-job-type":0} /tidb/cdc/default/default/task/position/512444c1-549e-43f6-8df5-a6a5f1b78368/66d7a011-45b1-479f-a14d-a97714ad2a87 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b13b93ac-ab2c-48a6-93ec-320e944238e8/66d7a011-45b1-479f-a14d-a97714ad2a87 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853887776047536 {"id":7365853887776047536,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/66d7a011-45b1-479f-a14d-a97714ad2a87 {UpstreamID:7365853887776047536 Namespace:default ID:66d7a011-45b1-479f-a14d-a97714ad2a87 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:55:18.127951184 +0800 CST StartTs:449576047217999875 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003170630 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576047243952131} {CheckpointTs:449576048882614285 MinTableBarrierTs:449576048882614285 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/512444c1-549e-43f6-8df5-a6a5f1b78368 {"id":"512444c1-549e-43f6-8df5-a6a5f1b78368","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996514} /tidb/cdc/default/__cdc_meta__/capture/b13b93ac-ab2c-48a6-93ec-320e944238e8 {"id":"b13b93ac-ab2c-48a6-93ec-320e944238e8","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996522} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1d11533 512444c1-549e-43f6-8df5-a6a5f1b78368 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1d115a7 b13b93ac-ab2c-48a6-93ec-320e944238e8 /tidb/cdc/default/default/changefeed/info/66d7a011-45b1-479f-a14d-a97714ad2a87 {"upstream-id":7365853887776047536,"namespace":"default","changefeed-id":"66d7a011-45b1-479f-a14d-a97714ad2a87","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:55:18.127951184+08:00","start-ts":449576047217999875,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576047243952131} /tidb/cdc/default/default/changefeed/status/66d7a011-45b1-479f-a14d-a97714ad2a87 {"checkpoint-ts":449576048882614285,"min-table-barrier-ts":449576048882614285,"admin-job-type":0} /tidb/cdc/default/default/task/position/512444c1-549e-43f6-8df5-a6a5f1b78368/66d7a011-45b1-479f-a14d-a97714ad2a87 {"checkpoint-ts":0,"resolved-ts":0,"co+ grep -q 'failed to get info:' unt":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b13b93ac-ab2c-48a6-93ec-320e944238e8/66d7a011-45b1-479f-a14d-a97714ad2a87 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853887776047536 {"id":7365853887776047536,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/66d7a011-45b1-479f-a14d-a97714ad2a87 {UpstreamID:7365853887776047536 Namespace:default ID:66d7a011-45b1-479f-a14d-a97714ad2a87 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:55:18.127951184 +0800 CST StartTs:449576047217999875 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003170630 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576047243952131} {CheckpointTs:449576048882614285 MinTableBarrierTs:449576048882614285 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/512444c1-549e-43f6-8df5-a6a5f1b78368 {"id":"512444c1-549e-43f6-8df5-a6a5f1b78368","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996514} /tidb/cdc/default/__cdc_meta__/capture/b13b93ac-ab2c-48a6-93ec-320e944238e8 {"id":"b13b93ac-ab2c-48a6-93ec-320e944238e8","address":"127.0.0.1:8302","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996522} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1d11533 512444c1-549e-43f6-8df5-a6a5f1b78368 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1d115a7 b13b93ac-ab2c-48a6-93ec-320e944238e8 /tidb/cdc/default/default/changefeed/info/66d7a011-45b1-479f-a14d-a97714ad2a87 {"upstream-id":7365853887776047536,"namespace":"default","changefeed-id":"66d7a011-45b1-479f-a14d-a97714ad2a87","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:55:18.127951184+08:00","start-ts":449576047217999875,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576047243952131} /tidb/cdc/default/default/changefeed/status/66d7a011-45b1-479f-a14d-a97714ad2a87 {"checkpoint-ts":449576048882614285,"min-table-barrier-ts":449576048882614285,"admin-job-type":0} /tidb/cdc/default/default/task/position/512444c1-549e-43f6-8df5-a6a5f1b78368/66d7a011-45b1-479f-a14d-a97714ad2a87 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b13b93ac-ab2c-48a6-93ec-320e944238e8/66d7a011-45b1-479f-a14d-a97714ad2a87 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853887776047536 {"id":7365853887776047536,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table processor_resolved_ts_fallback.t2 not exists for 1-th check, retry later table test.finish_mark not exists for 8-th check, retry later table processor_delay.t16 exists table processor_delay.t17 not exists for 1-th check, retry later [Mon May 6 19:55:25 CST 2024] <<<<<< START cdc server in http_api_tls_with_user_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates ']' + GO_FAILPOINTS= + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls_with_user_auth.3688236884.out server --log-file /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_http_api_tls_with_user_auth_tls2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_data_http_api_tls_with_user_auth_tls2 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls_with_user_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8301 --pd https://127.0.0.1:2579 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:55:25.986 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark19000] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 36-th check, retry later [2024/05/06 19:55:27.192 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] [2024/05/06 19:55:27.206 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark18000] [ts=449576049291034629] [2024/05/06 19:55:27.206 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark18000] [2024/05/06 19:55:27.206 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576049291034629] [2024/05/06 19:55:27.208 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:27.210 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:55:27.214 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:27.218 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:55:27.218 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark20000] table processor_resolved_ts_fallback.t2 not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.finish_mark not exists for 9-th check, retry later table processor_delay.t17 exists table processor_delay.t18 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/autorandom/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 19:55:28.000 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark19000] [2024/05/06 19:55:28.015 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark19000] [ts=449576049789108229] [2024/05/06 19:55:28.015 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark19000] [2024/05/06 19:55:28.015 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576049789108229] [2024/05/06 19:55:28.017 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:28.020 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:55:28.024 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:28.030 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:55:28.030 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 37-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:55:29 GMT < Content-Length: 1749 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38fb7bbf-acf0-48b1-8681-197eb5be4787 {"id":"38fb7bbf-acf0-48b1-8681-197eb5be4787","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996526} /tidb/cdc/default/__cdc_meta__/capture/531822ee-e263-4974-9812-add93d8c7a1c {"id":"531822ee-e263-4974-9812-add93d8c7a1c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996521} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dc2016631 531822ee-e263-4974-9812-add93d8c7a1c /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dc2016673 38fb7bbf-acf0-48b1-8681-197eb5be4787 /tidb/cdc/default/default/upstream/7365853933390517639 {"id":7365853933390517639,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38fb7bbf-acf0-48b1-8681-197eb5be4787 {"id":"38fb7bbf-acf0-48b1-8681-197eb5be4787","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996526} /tidb/cdc/default/__cdc_meta__/capture/531822ee-e263-4974-9812-add93d8c7a1c {"id":"531822ee-e263-4974-9812-add93d8c7a1c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996521} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dc2016631 531822ee-e263-4974-9812-add93d8c7a1c /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dc2016673 38fb7bbf-acf0-48b1-8681-197eb5be4787 /tidb/cdc/default/default/upstream/7365853933390517639 {"id":7365853933390517639,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38fb7bbf-acf0-48b1-8681-197eb5be4787 {"id":"38fb7bbf-acf0-48b1-8681-197eb5be4787","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996526} /tidb/cdc/default/__cdc_meta__/capture/531822ee-e263-4974-9812-add93d8c7a1c {"id":"531822ee-e263-4974-9812-add93d8c7a1c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996521} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dc2016631 531822ee-e263-4974-9812-add93d8c7a1c /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dc2016673 38fb7bbf-acf0-48b1-8681-197eb5be4787 /tidb/cdc/default/default/upstream/7365853933390517639 {"id":7365853933390517639,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + break + set +x [2024/05/06 19:55:29.218 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark20000] # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_nfs/sync_diff/output" source-instances = ["mysql1"] target-instance = "tidb0" target-check-tables = ["consistent_replicate_nfs.usertable*"] [data-sources] [data-sources.mysql1] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "449576046000603138" [data-sources.tidb0] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/05/06 19:55:29.422 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 19:55:29.423 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2024/05/06 19:55:29.423 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=:6060] [2024/05/06 19:55:29.423 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 19:55:29.423 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449576042225729608] [resolvedTs=449576046000603138] [2024/05/06 19:55:29.424 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=427.061ยตs] [2024/05/06 19:55:29.424 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 19:55:29.424 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 19:55:29.424 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/06 19:55:29.428 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 19:55:29.432 +08:00] [INFO] [mysql.go:197] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3708f0fc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-5k7gg, pid:6554, start at 2024-05-06 19:55:26.179974838 +0800 CST m=+5.212413892 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:26.186 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:26.143 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:26.143 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3708f0fc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-5k7gg, pid:6554, start at 2024-05-06 19:55:26.179974838 +0800 CST m=+5.212413892 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:26.186 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:26.143 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:26.143 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3708f340000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-5k7gg, pid:6639, start at 2024-05-06 19:55:26.30221143 +0800 CST m=+5.286788123 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:26.312 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:26.288 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:26.288 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table processor_resolved_ts_fallback.t2 exists table processor_resolved_ts_fallback.t3 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:55:29 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0041997a0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576047806775298 MinTableBarrierTs:449576047806775298 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d336479a-7d92-4da6-b6f0-70b1b9491b74 {"id":"d336479a-7d92-4da6-b6f0-70b1b9491b74","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996523} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e87c d336479a-7d92-4da6-b6f0-70b1b9491b74 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576047806775298,"min-table-barrier-ts":449576047806775298,"admin-job-type":0} /tidb/cdc/default/default/task/position/d336479a-7d92-4da6-b6f0-70b1b9491b74/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0041997a0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576047806775298 MinTableBarrierTs:449576047806775298 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d336479a-7d92-4da6-b6f0-70b1b9491b74 {"id":"d336479a-7d92-4da6-b6f0-70b1b9491b74","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996523} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e87c d336479a-7d92-4da6-b6f0-70b1b9491b74 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576047806775298,"min-table-barrier-ts":449576047806775298,"admin-job-type":0} /tidb/cdc/default/default/task/position/d336479a-7d92-4da6-b6f0-70b1b9491b74/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 + grep -q 'failed to get info:' {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0041997a0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576047806775298 MinTableBarrierTs:449576047806775298 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d336479a-7d92-4da6-b6f0-70b1b9491b74 {"id":"d336479a-7d92-4da6-b6f0-70b1b9491b74","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996523} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e87c d336479a-7d92-4da6-b6f0-70b1b9491b74 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576047806775298,"min-table-barrier-ts":449576047806775298,"admin-job-type":0} /tidb/cdc/default/default/task/position/d336479a-7d92-4da6-b6f0-70b1b9491b74/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' [2024/05/06 19:55:29.440 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:55:29.440 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:55:29.440 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:55:29.440 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:55:29.440 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:55:29.440 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:55:29.440 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:55:29.440 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:55:29.440 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:55:29.440 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:55:29.440 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:55:29.440 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:55:29.441 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:55:29.441 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:55:29.441 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:55:29.441 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:55:29.441 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 19:55:29.441 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 19:55:29.441 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 19:55:29.441 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 19:55:29.441 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 19:55:29.441 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 19:55:29.441 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 19:55:29.441 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 19:55:29.441 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 19:55:29.441 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 19:55:29.441 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 19:55:29.441 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 19:55:29.441 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 19:55:29.441 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 19:55:29.441 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 19:55:29.441 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 19:55:29.441 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 19:55:29.441 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 19:55:29.441 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/06 19:55:29.445 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 19:55:29.447 +08:00] [INFO] [mysql_ddl_sink.go:105] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/06 19:55:29.454 +08:00] [INFO] [file.go:284] ["ignore logs which commitTs is greater than resolvedTs"] [filename=127.0.0.1:8300_9111f490-bb4a-4925-a222-2b4e77f27909_row_449576046629486679_3c44ad80-fa1b-4982-80a4-fd13d2565d8b.log] [endTs=449576046000603138] [2024/05/06 19:55:29.472 +08:00] [INFO] [file.go:284] ["ignore logs which commitTs is greater than resolvedTs"] [filename=127.0.0.1:8300_9111f490-bb4a-4925-a222-2b4e77f27909_row_449576046000603139_7a4c3e8e-02d1-47f8-9682-fc81fdb3aa23.log] [endTs=449576046000603138] [2024/05/06 19:55:29.479 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=56.157661ms] table test.finish_mark not exists for 10-th check, retry later table processor_delay.t18 exists table processor_delay.t19 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_state http://127.0.0.1:2379 84dd948a-181c-44bb-b6db-5aba235f39d2 finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=84dd948a-181c-44bb-b6db-5aba235f39d2 + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 84dd948a-181c-44bb-b6db-5aba235f39d2 -s + info='{ "upstream_id": 7365853500178616945, "namespace": "default", "id": "84dd948a-181c-44bb-b6db-5aba235f39d2", "state": "finished", "checkpoint_tso": 449576047594700804, "checkpoint_time": "2024-05-06 19:55:19.450", "error": null }' + echo '{ "upstream_id": 7365853500178616945, "namespace": "default", "id": "84dd948a-181c-44bb-b6db-5aba235f39d2", "state": "finished", "checkpoint_tso": 449576047594700804, "checkpoint_time": "2024-05-06 19:55:19.450", "error": null }' { "upstream_id": 7365853500178616945, "namespace": "default", "id": "84dd948a-181c-44bb-b6db-5aba235f39d2", "state": "finished", "checkpoint_tso": 449576047594700804, "checkpoint_time": "2024-05-06 19:55:19.450", "error": null } ++ echo '{' '"upstream_id":' 7365853500178616945, '"namespace":' '"default",' '"id":' '"84dd948a-181c-44bb-b6db-5aba235f39d2",' '"state":' '"finished",' '"checkpoint_tso":' 449576047594700804, '"checkpoint_time":' '"2024-05-06' '19:55:19.450",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365853500178616945, '"namespace":' '"default",' '"id":' '"84dd948a-181c-44bb-b6db-5aba235f39d2",' '"state":' '"finished",' '"checkpoint_tso":' 449576047594700804, '"checkpoint_time":' '"2024-05-06' '19:55:19.450",' '"error":' null '}' + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:55:26 CST 2024] <<<<<< run test case changefeed_finish success! >>>>>> table mark.finish_mark_3 not exists for 38-th check, retry later [2024/05/06 19:55:31.231 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark20000] [2024/05/06 19:55:31.246 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark20000] [ts=449576050339872773] [2024/05/06 19:55:31.246 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark20000] [2024/05/06 19:55:31.246 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449576050339872773] [2024/05/06 19:55:31.248 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:31.251 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:55:31.255 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 19:55:31.259 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 19:55:31.259 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] [2024/05/06 19:55:31.259 +08:00] [INFO] [case.go:615] ["bank test finished"] [valid=20] [tries=20] [ratio=1] [Mon May 6 19:55:31 CST 2024] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.79637965.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_resolved_ts_fallback.t3 exists cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^2$' 2 run task successfully + cleanup_process cdc.test "is-owner": true, run task successfully [2024/05/06 19:55:31.224 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}] [checkpointTs=449576046000603138] [2024/05/06 19:55:31.224 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}] [checkpointTs=449576046000603138] [2024/05/06 19:55:31.324 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=17930] [appliedDDLCount=0] [currentCheckpoint=449576046000603138] [2024/05/06 19:55:31.324 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 19:55:31.324 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/06 19:55:31.324 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 19:55:31.324 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/06 19:55:31.324 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 19:55:31.325 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/06 19:55:31.325 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 19:55:31.325 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 19:55:31.325 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/06 19:55:31.325 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 19:55:31.325 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 19:55:31.324 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 19:55:31.325 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 19:55:31.325 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 19:55:31.325 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/06 19:55:31.325 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] Apply redo log successfully table test.finish_mark not exists for 11-th check, retry later table processor_delay.t19 exists table processor_delay.t20 not exists for 1-th check, retry later killing cdc1: 5754 cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff successfully [Mon May 6 19:55:31 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> count(*) 44926 pass test: check health pass test: get status wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... pass test: create changefeed check_changefeed_state https://127.0.0.1:2579 changefeed-test1 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test1 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test1 -s owner pid: 4407 owner id d336479a-7d92-4da6-b6f0-70b1b9491b74 [Mon May 6 19:55:32 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.45354537.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 3-th time... + info='{ "upstream_id": 7365853933390517639, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 449576050829557763, "checkpoint_time": "2024-05-06 19:55:31.790", "error": null }' + echo '{ "upstream_id": 7365853933390517639, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 449576050829557763, "checkpoint_time": "2024-05-06 19:55:31.790", "error": null }' { "upstream_id": 7365853933390517639, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 449576050829557763, "checkpoint_time": "2024-05-06 19:55:31.790", "error": null } ++ echo '{' '"upstream_id":' 7365853933390517639, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 449576050829557763, '"checkpoint_time":' '"2024-05-06' '19:55:31.790",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365853933390517639, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 449576050829557763, '"checkpoint_time":' '"2024-05-06' '19:55:31.790",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test2 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test2 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test2 -s table mark.finish_mark_3 not exists for 39-th check, retry later + info='{ "upstream_id": 7365853933390517639, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 449576050895093761, "checkpoint_time": "2024-05-06 19:55:32.040", "error": null }' + echo '{ "upstream_id": 7365853933390517639, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 449576050895093761, "checkpoint_time": "2024-05-06 19:55:32.040", "error": null }' { "upstream_id": 7365853933390517639, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 449576050895093761, "checkpoint_time": "2024-05-06 19:55:32.040", "error": null } ++ echo '{' '"upstream_id":' 7365853933390517639, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 449576050895093761, '"checkpoint_time":' '"2024-05-06' '19:55:32.040",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365853933390517639, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 449576050895093761, '"checkpoint_time":' '"2024-05-06' '19:55:32.040",' '"error":' null '}' ++ jq -r .error.message table test.finish_mark not exists for 12-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit ++ date + echo '[Mon May 6 19:55:33 CST 2024] <<<<<< run test case bank success! >>>>>>' [Mon May 6 19:55:33 CST 2024] <<<<<< run test case bank success! >>>>>> + stop_tidb_cluster + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test3 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test3 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test3 -s table processor_delay.t20 exists table processor_delay.t21 not exists for 1-th check, retry later + info='{ "upstream_id": 7365853933390517639, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 449576051248988169, "checkpoint_time": "2024-05-06 19:55:33.390", "error": null }' + echo '{ "upstream_id": 7365853933390517639, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 449576051248988169, "checkpoint_time": "2024-05-06 19:55:33.390", "error": null }' { "upstream_id": 7365853933390517639, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 449576051248988169, "checkpoint_time": "2024-05-06 19:55:33.390", "error": null } ++ echo '{' '"upstream_id":' 7365853933390517639, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 449576051248988169, '"checkpoint_time":' '"2024-05-06' '19:55:33.390",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365853933390517639, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 449576051248988169, '"checkpoint_time":' '"2024-05-06' '19:55:33.390",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test4 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test4 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test4 -s + info='{ "upstream_id": 7365853933390517639, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 449576051248988169, "checkpoint_time": "2024-05-06 19:55:33.390", "error": null }' + echo '{ "upstream_id": 7365853933390517639, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 449576051248988169, "checkpoint_time": "2024-05-06 19:55:33.390", "error": null }' { "upstream_id": 7365853933390517639, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 449576051248988169, "checkpoint_time": "2024-05-06 19:55:33.390", "error": null } ++ echo '{' '"upstream_id":' 7365853933390517639, '"namespace":' '"default",' '"id":' '"changefeed-test4",' '"state":' '"normal",' '"checkpoint_tso":' 449576051248988169, '"checkpoint_time":' '"2024-05-06' '19:55:33.390",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365853933390517639, '"namespace":' '"default",' '"id":' '"changefeed-test4",' '"state":' '"normal",' '"checkpoint_tso":' 449576051248988169, '"checkpoint_time":' '"2024-05-06' '19:55:33.390",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37095864000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-rqt30, pid:6574, start at 2024-05-06 19:55:32.777080713 +0800 CST m=+26.732373285 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:32.783 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:32.761 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:32.761 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37095864000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-rqt30, pid:6574, start at 2024-05-06 19:55:32.777080713 +0800 CST m=+26.732373285 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:32.783 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:32.761 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:32.761 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370827940008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-rqt30, pid:6660, start at 2024-05-06 19:55:13.262165129 +0800 CST m=+7.141554846 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:13.270 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:13.253 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:13.253 +0800 All versions after safe point can be accessed. (DO NOT EDIT) cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 2-th time, retry later Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:55:34 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/205d8404-9530-4461-8b38-91cff5275bfc {"id":"205d8404-9530-4461-8b38-91cff5275bfc","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996531} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc2101737 205d8404-9530-4461-8b38-91cff5275bfc /tidb/cdc/default/default/upstream/7365853948448107245 {"id":7365853948448107245,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/205d8404-9530-4461-8b38-91cff5275bfc {"id":"205d8404-9530-4461-8b38-91cff5275bfc","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996531} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc2101737 205d8404-9530-4461-8b38-91cff5275bfc /tidb/cdc/default/default/upstream/7365853948448107245 {"id":7365853948448107245,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/205d8404-9530-4461-8b38-91cff5275bfc {"id":"205d8404-9530-4461-8b38-91cff5275bfc","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996531} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc2101737 205d8404-9530-4461-8b38-91cff5275bfc /tidb/cdc/default/default/upstream/7365853948448107245 {"id":7365853948448107245,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table mark.finish_mark_3 not exists for 40-th check, retry later table test.finish_mark not exists for 13-th check, retry later table processor_delay.t21 exists table processor_delay.t22 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370976340018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:14377, start at 2024-05-06 19:55:34.715863149 +0800 CST m=+5.162114898 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:34.724 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:34.720 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:34.720 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370976340018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:14377, start at 2024-05-06 19:55:34.715863149 +0800 CST m=+5.162114898 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:34.724 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:34.720 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:34.720 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370979440014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:14458, start at 2024-05-06 19:55:34.88387565 +0800 CST m=+5.282181364 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:34.894 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:34.865 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:34.865 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 19:55:35.818 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc0016e65a0/127.0.0.1:2379] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"] table test.table1 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:55:35 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003adea20 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576051738935301 MinTableBarrierTs:449576051738935301 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d3933bd-621f-44ce-82b5-9c7a89068363 {"id":"0d3933bd-621f-44ce-82b5-9c7a89068363","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996532} /tidb/cdc/default/__cdc_meta__/capture/d336479a-7d92-4da6-b6f0-70b1b9491b74 {"id":"d336479a-7d92-4da6-b6f0-70b1b9491b74","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996523} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e87c d336479a-7d92-4da6-b6f0-70b1b9491b74 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e8e0 0d3933bd-621f-44ce-82b5-9c7a89068363 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576051738935301,"min-table-barrier-ts":449576051738935301,"admin-job-type":0} /tidb/cdc/default/default/task/position/0d3933bd-621f-44ce-82b5-9c7a89068363/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d336479a-7d92-4da6-b6f0-70b1b9491b74/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003adea20 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576051738935301 MinTableBarrierTs:449576051738935301 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d3933bd-621f-44ce-82b5-9c7a89068363 {"id":"0d3933bd-621f-44ce-82b5-9c7a89068363","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996532} /tidb/cdc/default/__cdc_meta__/capture/d336479a-7d92-4da6-b6f0-70b1b9491b74 {"id":"d336479a-7d92-4da6-b6f0-70b1b9491b74","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996523} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e87c d336479a-7d92-4da6-b6f0-70b1b9491b74 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e8e0 0d3933bd-621f-44ce-82b5-9c7a89068363 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576051738935301,"min-table-barrier-ts":449576051738935301,"admin-job-type":0} /tidb/cdc/default/default/task/position/0d3933bd-621f-44ce-82b5-9c7a89068363/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'failed to get info:' /tidb/cdc/default/default/task/position/d336479a-7d92-4da6-b6f0-70b1b9491b74/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003adea20 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576051738935301 MinTableBarrierTs:449576051738935301 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d3933bd-621f-44ce-82b5-9c7a89068363 {"id":"0d3933bd-621f-44ce-82b5-9c7a89068363","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996532} /tidb/cdc/default/__cdc_meta__/capture/d336479a-7d92-4da6-b6f0-70b1b9491b74 {"id":"d336479a-7d92-4da6-b6f0-70b1b9491b74","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996523} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e87c d336479a-7d92-4da6-b6f0-70b1b9491b74 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e8e0 0d3933bd-621f-44ce-82b5-9c7a89068363 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576051738935301,"min-table-barrier-ts":449576051738935301,"admin-job-type":0} /tidb/cdc/default/default/task/position/0d3933bd-621f-44ce-82b5-9c7a89068363/15d744e6-9+ grep -q 'etcd info' 542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d336479a-7d92-4da6-b6f0-70b1b9491b74/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "d336479a-7d92-4da6-b6f0-70b1b9491b74" | grep id ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 41-th check, retry later [Mon May 6 19:55:36 CST 2024] <<<<<< START cdc server in multi_rocks case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.1954919551.out server --log-file /tmp/tidb_cdc_test/multi_rocks/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_rocks/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 "cluster-id": "default" "id": "0d3933bd-621f-44ce-82b5-9c7a89068363", "cluster-id": "default" run task successfully table test.simple1 exists running list_changefeed pass test: list changefeed + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.15762.out cli tso query --pd=http://127.0.0.1:2379 table test.finish_mark not exists for 14-th check, retry later table test.table1 exists table test.table2 not exists for 1-th check, retry later table processor_delay.t22 exists table processor_delay.t23 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 3-th time, retry later running get_changefeed pass test: get changefeed capture_id: 0d3933bd-621f-44ce-82b5-9c7a89068363 table mark.finish_mark_3 not exists for 42-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:55:39 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + set +x + tso='449576052502036481 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576052502036481 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:55:39 CST 2024] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.1579315795.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/181a533c-7b59-4db0-8022-e7168777281e {"id":"181a533c-7b59-4db0-8022-e7168777281e","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996536} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1d9db67 181a533c-7b59-4db0-8022-e7168777281e /tidb/cdc/default/default/upstream/7365853891061512353 {"id":7365853891061512353,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/181a533c-7b59-4db0-8022-e7168777281e {"id":"181a533c-7b59-4db0-8022-e7168777281e","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996536} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1d9db67 181a533c-7b59-4db0-8022-e7168777281e /tidb/cdc/default/default/upstream/7365853891061512353 {"id":7365853891061512353,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/181a533c-7b59-4db0-8022-e7168777281e {"id":"181a533c-7b59-4db0-8022-e7168777281e","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996536} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc1d9db67 181a533c-7b59-4db0-8022-e7168777281e /tidb/cdc/default/default/upstream/7365853891061512353 {"id":7365853891061512353,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.cli.19828.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ table test.finish_mark not exists for 15-th check, retry later table test.table2 exists table processor_delay.t23 exists table processor_delay.t24 not exists for 1-th check, retry later running pause_changefeed Create changefeed successfully! ID: f74f8f79-6882-429d-9907-8d62d34fdb89 Info: {"upstream_id":7365853891061512353,"namespace":"default","id":"f74f8f79-6882-429d-9907-8d62d34fdb89","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:55:39.949823019+08:00","start_ts":449576052945321986,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576052945321986,"checkpoint_ts":449576052945321986,"checkpoint_time":"2024-05-06 19:55:39.861"} PASS ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later coverage: 2.6% of statements in github.com/pingcap/tiflow/... pass test: pause changefeed =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 43-th check, retry later running update_changefeed + set +x ***************** properties ***************** "workload"="core" "requestdistribution"="uniform" "mysql.port"="4000" "dotransactions"="false" "mysql.db"="multi_rocks" "table"="a1" "updateproportion"="0" "readallfields"="true" "mysql.user"="root" "mysql.host"="127.0.0.1" "readproportion"="0" "operationcount"="0" "threadcount"="2" "scanproportion"="0" "insertproportion"="0" "recordcount"="1000" ********************************************** table test.finish_mark not exists for 16-th check, retry later table processor_delay.t24 exists table processor_delay.t25 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... pass test: update changefeed Run finished, takes 551.618812ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1857.0, Avg(us): 1074, Min(us): 832, Max(us): 13110, 95th(us): 2000, 99th(us): 2000 nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later ***************** properties ***************** "table"="a2" "requestdistribution"="uniform" "threadcount"="2" "readproportion"="0" "mysql.port"="4000" "updateproportion"="0" "mysql.host"="127.0.0.1" "mysql.user"="root" "mysql.db"="multi_rocks" "workload"="core" "readallfields"="true" "insertproportion"="0" "scanproportion"="0" "dotransactions"="false" "recordcount"="1000" "operationcount"="0" ********************************************** VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3709bb900014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc, pid:30111, start at 2024-05-06 19:55:39.15503704 +0800 CST m=+5.303484317 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:39.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:39.158 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:39.158 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3709bb900014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc, pid:30111, start at 2024-05-06 19:55:39.15503704 +0800 CST m=+5.303484317 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:39.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:39.158 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:39.158 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3709bd040014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc, pid:30184, start at 2024-05-06 19:55:39.23025528 +0800 CST m=+5.304499058 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:39.239 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:39.201 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:39.201 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/autorandom/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/autorandom/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully running resume_changefeed Run finished, takes 551.336284ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1873.7, Avg(us): 1072, Min(us): 794, Max(us): 17520, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "dotransactions"="false" "scanproportion"="0" "mysql.db"="multi_rocks" "updateproportion"="0" "mysql.host"="127.0.0.1" "threadcount"="2" "readallfields"="true" "operationcount"="0" "table"="a3" "insertproportion"="0" "workload"="core" "readproportion"="0" "recordcount"="1000" "mysql.user"="root" "mysql.port"="4000" "requestdistribution"="uniform" ********************************************** + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:55:42 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/700460df-b9ba-43fa-b4bf-aaf16c06a6a0 {"id":"700460df-b9ba-43fa-b4bf-aaf16c06a6a0","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996539} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc23b8c34 700460df-b9ba-43fa-b4bf-aaf16c06a6a0 /tidb/cdc/default/default/upstream/7365853993564679210 {"id":7365853993564679210,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/700460df-b9ba-43fa-b4bf-aaf16c06a6a0 {"id":"700460df-b9ba-43fa-b4bf-aaf16c06a6a0","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996539} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc23b8c34 700460df-b9ba-43fa-b4bf-aaf16c06a6a0 /tidb/cdc/default/default/upstream/7365853993564679210 {"id":7365853993564679210,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/700460df-b9ba-43fa-b4bf-aaf16c06a6a0 {"id":"700460df-b9ba-43fa-b4bf-aaf16c06a6a0","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996539} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc23b8c34 700460df-b9ba-43fa-b4bf-aaf16c06a6a0 /tidb/cdc/default/default/upstream/7365853993564679210 {"id":7365853993564679210,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.15850.out cli changefeed create --start-ts=449576052502036481 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ table mark.finish_mark_3 not exists for 44-th check, retry later Create changefeed successfully! ID: a4883192-3ced-4bb8-8fff-d5529a7ef0ba Info: {"upstream_id":7365853993564679210,"namespace":"default","id":"a4883192-3ced-4bb8-8fff-d5529a7ef0ba","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:55:43.162131476+08:00","start_ts":449576052502036481,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576052502036481,"checkpoint_ts":449576052502036481,"checkpoint_time":"2024-05-06 19:55:38.170"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kill_owner_with_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... pass test: resume changefeed start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... table test.finish_mark not exists for 17-th check, retry later Run finished, takes 516.008249ms INSERT - Takes(s): 0.5, Count: 999, OPS: 2000.7, Avg(us): 987, Min(us): 773, Max(us): 16559, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "threadcount"="2" "readallfields"="true" "readproportion"="0" "table"="a4" "dotransactions"="false" "insertproportion"="0" "requestdistribution"="uniform" "workload"="core" "recordcount"="1000" "updateproportion"="0" "scanproportion"="0" "mysql.port"="4000" "operationcount"="0" "mysql.host"="127.0.0.1" "mysql.user"="root" "mysql.db"="multi_rocks" ********************************************** table processor_delay.t25 exists table processor_delay.t26 not exists for 1-th check, retry later Run finished, takes 493.078454ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 2097.8, Avg(us): 958, Min(us): 762, Max(us): 16326, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "updateproportion"="0" "mysql.db"="multi_rocks" "dotransactions"="false" "operationcount"="0" "requestdistribution"="uniform" "mysql.port"="4000" "workload"="core" "readproportion"="0" "table"="a5" "recordcount"="1000" "threadcount"="2" "mysql.host"="127.0.0.1" "scanproportion"="0" "mysql.user"="root" "insertproportion"="0" "readallfields"="true" ********************************************** nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [Mon May 6 19:55:44 CST 2024] <<<<<< START cdc server in autorandom case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.autorandom.3150531507.out server --log-file /tmp/tidb_cdc_test/autorandom/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/autorandom/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x running rebalance_table pass test: rebalance table cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' 1 run task successfully check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... Run finished, takes 885.071059ms INSERT - Takes(s): 0.9, Count: 1000, OPS: 1150.5, Avg(us): 1738, Min(us): 809, Max(us): 78435, 95th(us): 2000, 99th(us): 16000 table mark.finish_mark_3 not exists for 45-th check, retry later Resume changefeed with checkpointTs3 449576053910798342 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2590 0 --:--:-- --:--:-- --:--:-- 2600 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 19:55:31.811","puller_resolved_ts":"2024-05-06 19:55:24.811","last_synced_ts":"2024-05-06 19:53:16.410","now_ts":"2024-05-06 19:55:33.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:55:31.811","puller_resolved_ts":"2024-05-06' '19:55:24.811","last_synced_ts":"2024-05-06' '19:53:16.410","now_ts":"2024-05-06' '19:55:33.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 9413 8.6 0.0 14532136 143576 ? Sl 19:52 0:13 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 9477 5.9 0.0 14467560 138216 ? Sl 19:52 0:09 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1714996538.9440129,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002a55dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714996538.94411,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714996539.025086,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0041f0380/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714996539.0251844,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714996539.8775995,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0016e5500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714996539.877677,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T19:55:43.717544+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e69180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:55:43.72015+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e53180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:55:43.803392+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0009ef500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table savepoint.finish_mark not exists for 1-th check, retry later Verifying downstream PD is started... table test.table5 not exists for 1-th check, retry later table test.finish_mark not exists for 18-th check, retry later table processor_delay.t26 exists table processor_delay.t27 not exists for 1-th check, retry later table multi_rocks.finish_mark not exists for 1-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later running list_processor pass test: list processors start tidb cluster in /tmp/tidb_cdc_test/kill_owner_with_ddl Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... check diff successfully table mark.finish_mark_3 not exists for 46-th check, retry later wait process cdc.test exit for 1-th time... running get_processor table savepoint.finish_mark not exists for 2-th check, retry later table test.table5 not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:55:47 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e584929e-20b6-4c72-aca8-6d6bee40bda8 {"id":"e584929e-20b6-4c72-aca8-6d6bee40bda8","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996544} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc2486433 e584929e-20b6-4c72-aca8-6d6bee40bda8 /tidb/cdc/default/default/upstream/7365854013143649720 {"id":7365854013143649720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e584929e-20b6-4c72-aca8-6d6bee40bda8 {"id":"e584929e-20b6-4c72-aca8-6d6bee40bda8","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996544} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc2486433 e584929e-20b6-4c72-aca8-6d6bee40bda8 /tidb/cdc/default/default/upstream/7365854013143649720 {"id":7365854013143649720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e584929e-20b6-4c72-aca8-6d6bee40bda8 {"id":"e584929e-20b6-4c72-aca8-6d6bee40bda8","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996544} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc2486433 e584929e-20b6-4c72-aca8-6d6bee40bda8 /tidb/cdc/default/default/upstream/7365854013143649720 {"id":7365854013143649720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 0daabee2-1be3-491e-97b2-bbbd82741b8f Info: {"upstream_id":7365854013143649720,"namespace":"default","id":"0daabee2-1be3-491e-97b2-bbbd82741b8f","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:55:47.464406428+08:00","start_ts":449576054910615554,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576054910615554,"checkpoint_ts":449576054910615554,"checkpoint_time":"2024-05-06 19:55:47.358"} wait process cdc.test exit for 3-th time... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release empty select id, val from test.availability1 where id=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table test.finish_mark not exists for 19-th check, retry later table processor_delay.t27 exists table processor_delay.t28 not exists for 1-th check, retry later table multi_rocks.finish_mark not exists for 2-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:55:48 CST 2024] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> table autorandom_test.table_a not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 47-th check, retry later table savepoint.finish_mark exists table test.table5 not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff successfully Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: get processor Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 20-th check, retry later table processor_delay.t28 exists table processor_delay.t29 not exists for 1-th check, retry later table multi_rocks.finish_mark not exists for 3-th check, retry later wait process cdc.test exit for 2-th time... {"level":"warn","ts":"2024-05-06T19:55:49.719086+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e69180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:55:49.722089+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e53180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:55:49.804949+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0009ef500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} nonempty select id, val from test.availability2 where id=1 and val=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later running move_table wait process cdc.test exit for 3-th time... pass test: move table table autorandom_test.table_a exists check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:55:51 CST 2024] <<<<<< run test case savepoint success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 48-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.table5 exists table test.table6 not exists for 1-th check, retry later table test.finish_mark not exists for 21-th check, retry later table processor_delay.t29 exists table processor_delay.t30 not exists for 1-th check, retry later table multi_rocks.finish_mark not exists for 4-th check, retry later running set_log_level pass test: set log level nonempty select id, val from test.availability2 where id=1 and val=22 run task successfully empty select id, val from test.availability2 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) running remove_changefeed ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 49-th check, retry later wait process cdc.test exit for 1-th time... table test.table6 exists check diff successfully table test.finish_mark not exists for 22-th check, retry later pass test: remove changefeed table processor_delay.t30 exists table processor_delay.t31 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... Resume changefeed with checkpointTs1 449576051577716743 table multi_rocks.finish_mark not exists for 5-th check, retry later table test.table3 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) empty select id, val from test.availability2 where id=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later wait process cdc.test exit for 3-th time... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:55:54 CST 2024] <<<<<< run test case autorandom success! >>>>>> running resign_owner pass test: resign owner table mark.finish_mark_3 not exists for 50-th check, retry later table test.finish_mark not exists for 23-th check, retry later table processor_delay.t31 exists table processor_delay.t32 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:55:55.720576+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e69180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:55:55.722969+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e53180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} table multi_rocks.finish_mark not exists for 6-th check, retry later running get_tso ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:55:55.805958+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0009ef500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} nonempty select id, val from test.availability3 where id=1 and val=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later pass test: get tso [2024/05/06 19:55:55.355 +08:00] [INFO] [case.go:115] ["sync updatePKUK take: 12.310851462s"] table test.table3 not exists for 2-th check, retry later table mark.finish_mark_3 not exists for 51-th check, retry later running verify_table VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370ac1180013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-59v7t, pid:7091, start at 2024-05-06 19:55:55.8921567 +0800 CST m=+5.155345075 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:55.901 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:55.897 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:55.897 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370ac1180013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-59v7t, pid:7091, start at 2024-05-06 19:55:55.8921567 +0800 CST m=+5.155345075 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:55.901 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:55.897 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:55.897 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370ac1c00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-59v7t, pid:7181, start at 2024-05-06 19:55:55.915570511 +0800 CST m=+5.129788335 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:55.922 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:55.888 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:55.888 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 24-th check, retry later pass test: verify table VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370aaa1c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:9859, start at 2024-05-06 19:55:54.415773247 +0800 CST m=+5.295149964 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:54.423 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:54.425 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:54.425 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370aaa1c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:9859, start at 2024-05-06 19:55:54.415773247 +0800 CST m=+5.295149964 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:54.423 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:54.425 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:54.425 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370aa9fc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:9935, start at 2024-05-06 19:55:54.402742882 +0800 CST m=+5.218711879 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:54.410 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:54.367 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:54.367 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table multi_rocks.finish_mark exists VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370ab5400009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-98cvh, pid:3970, start at 2024-05-06 19:55:55.097872603 +0800 CST m=+5.689138539 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:55.107 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:55.088 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:55.088 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370ab5400009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-98cvh, pid:3970, start at 2024-05-06 19:55:55.097872603 +0800 CST m=+5.689138539 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:55.107 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:55.088 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:55.088 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370ab74c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-98cvh, pid:4038, start at 2024-05-06 19:55:55.247946702 +0800 CST m=+5.786369081 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:57:55.258 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:55:55.219 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:45:55.219 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully table processor_delay.t32 exists table processor_delay.t33 not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } running create_changefeed_v2 table test.table3 not exists for 3-th check, retry later wait process cdc.test exit for 1-th time... Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) nonempty select id, val from test.availability3 where id=1 and val=22 run task successfully empty select id, val from test.availability3 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later wait process cdc.test exit for 2-th time... [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials pass test: create changefeed v2 [Pipeline] } [Pipeline] // timeout [Pipeline] } table mark.finish_mark_3 not exists for 52-th check, retry later [Pipeline] // stage [Pipeline] } [Mon May 6 19:55:59 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.84158417.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 3-th time... [Pipeline] // container [Pipeline] } [Mon May 6 19:55:59 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file.1123911241.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:55:59 CST 2024] <<<<<< run test case multi_rocks success! >>>>>> [Pipeline] // withEnv [Pipeline] } table test.finish_mark not exists for 25-th check, retry later [Pipeline] // stage table processor_delay.t33 exists table processor_delay.t34 not exists for 1-th check, retry later [Pipeline] } table test.table3 not exists for 4-th check, retry later [Mon May 6 19:55:59 CST 2024] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.53895391.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 running delete_changefeed_v2 empty select id, val from test.availability3 where id=1 run task successfully test_gap_between_watch_capture pass wait process cdc.test exit for 1-th time... pass test: remove changefeed v2 table mark.finish_mark_3 not exists for 53-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status + source /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:56:01.721581+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e69180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} table test.finish_mark not exists for 26-th check, retry later {"level":"warn","ts":"2024-05-06T19:56:01.724747+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e53180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} running unsafe_apis table processor_delay.t34 exists table processor_delay.t35 not exists for 1-th check, retry later wait process cdc.test exit for 3-th time... 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:56:01.807396+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0009ef500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} status code 200 pass test: show metadata status code 200 status code 200 pass test: delete service_gc_safepoint status code 500 pass test: resolve lock table test.table3 exists table test.table4 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit run test case delete_owner_key [Mon May 6 19:56:02 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.49564958.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:56:02 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/54ac2e1b-6966-4d4a-b833-ea39887b7134 {"id":"54ac2e1b-6966-4d4a-b833-ea39887b7134","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996559} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc28a5c2b 54ac2e1b-6966-4d4a-b833-ea39887b7134 /tidb/cdc/default/default/upstream/7365854083392535616 {"id":7365854083392535616,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/54ac2e1b-6966-4d4a-b833-ea39887b7134 {"id":"54ac2e1b-6966-4d4a-b833-ea39887b7134","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996559} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc28a5c2b 54ac2e1b-6966-4d4a-b833-ea39887b7134 /tidb/cdc/default/default/upstream/7365854083392535616 {"id":7365854083392535616,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/54ac2e1b-6966-4d4a-b833-ea39887b7134 {"id":"54ac2e1b-6966-4d4a-b833-ea39887b7134","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996559} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc28a5c2b 54ac2e1b-6966-4d4a-b833-ea39887b7134 /tidb/cdc/default/default/upstream/7365854083392535616 {"id":7365854083392535616,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 3c3d0780-7268-4b21-8097-58d5a890cac7 Info: {"upstream_id":7365854083392535616,"namespace":"default","id":"3c3d0780-7268-4b21-8097-58d5a890cac7","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:56:02.307306078+08:00","start_ts":449576058800570371,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576058800570371,"checkpoint_ts":449576058800570371,"checkpoint_time":"2024-05-06 19:56:02.197"} table kill_owner_with_ddl.t1 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:56:02 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dfa3701f-a9d2-4186-88b5-0c2175bce45b {"id":"dfa3701f-a9d2-4186-88b5-0c2175bce45b","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996559} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc2840732 dfa3701f-a9d2-4186-88b5-0c2175bce45b /tidb/cdc/default/default/upstream/7365854079682308468 {"id":7365854079682308468,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dfa3701f-a9d2-4186-88b5-0c2175bce45b {"id":"dfa3701f-a9d2-4186-88b5-0c2175bce45b","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996559} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc2840732 dfa3701f-a9d2-4186-88b5-0c2175bce45b /tidb/cdc/default/default/upstream/7365854079682308468 {"id":7365854079682308468,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dfa3701f-a9d2-4186-88b5-0c2175bce45b {"id":"dfa3701f-a9d2-4186-88b5-0c2175bce45b","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996559} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc2840732 dfa3701f-a9d2-4186-88b5-0c2175bce45b /tidb/cdc/default/default/upstream/7365854079682308468 {"id":7365854079682308468,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table mark.finish_mark_3 not exists for 54-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:56:03 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2a649a60-d5ed-43f6-af47-a74dddc38ee7 {"id":"2a649a60-d5ed-43f6-af47-a74dddc38ee7","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996560} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc27f3137 2a649a60-d5ed-43f6-af47-a74dddc38ee7 /tidb/cdc/default/default/upstream/7365854070357686612 {"id":7365854070357686612,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2a649a60-d5ed-43f6-af47-a74dddc38ee7 {"id":"2a649a60-d5ed-43f6-af47-a74dddc38ee7","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996560} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc27f3137 2a649a60-d5ed-43f6-af47-a74dddc38ee7 /tidb/cdc/default/default/upstream/7365854070357686612 {"id":7365854070357686612,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2a649a60-d5ed-43f6-af47-a74dddc38ee7 {"id":"2a649a60-d5ed-43f6-af47-a74dddc38ee7","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996560} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc27f3137 2a649a60-d5ed-43f6-af47-a74dddc38ee7 /tidb/cdc/default/default/upstream/7365854070357686612 {"id":7365854070357686612,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: ce624732-a136-48a7-a1d1-ab6fd522b642 Info: {"upstream_id":7365854070357686612,"namespace":"default","id":"ce624732-a136-48a7-a1d1-ab6fd522b642","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-05-06T19:56:03.281767366+08:00","start_ts":449576058208387073,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576058208387073,"checkpoint_ts":449576058208387073,"checkpoint_time":"2024-05-06 19:55:59.938"} table test.finish_mark not exists for 27-th check, retry later table processor_delay.t35 exists table processor_delay.t36 not exists for 1-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:56:03.709202+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e69180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-06T19:56:03.709258+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T19:56:03.711544+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e53180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-06T19:56:03.711604+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:56:03.793702+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0009ef500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-06T19:56:03.793758+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} wait process cdc.test exit for 1-th time... table test.table4 exists ***************** properties ***************** "mysql.host"="127.0.0.1" "workload"="core" "operationcount"="0" "mysql.port"="4000" "scanproportion"="0" "fieldcount"="100" "mysql.db"="consistent_replicate_storage_file" "dotransactions"="false" "updateproportion"="0" "recordcount"="5000" "mysql.user"="root" "readallfields"="true" "readproportion"="0" "insertproportion"="0" "requestdistribution"="uniform" "threadcount"="10" ********************************************** wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... check diff successfully wait process cdc.test exit for 3-th time... wait process cdc.test exit for 1-th time... table kill_owner_with_ddl.t1 exists wait process 8420 exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:56:05 CST 2024] <<<<<< run test case http_api_tls_with_user_auth success! >>>>>> table mark.finish_mark_3 not exists for 55-th check, retry later wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:56:05 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0026beea0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576058030391302 MinTableBarrierTs:449576059603255302 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576058030391302, checkpointTs: 449576058030391302, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576058030391302, checkpointTs: 449576058030391302, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576058030391302, checkpointTs: 449576058030391302, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/931a4a3b-2498-4db4-8736-764b7cf44870 {"id":"931a4a3b-2498-4db4-8736-764b7cf44870","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996562} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e9ec 931a4a3b-2498-4db4-8736-764b7cf44870 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576058030391302,"min-table-barrier-ts":449576059603255302,"admin-job-type":0} /tidb/cdc/default/default/task/position/931a4a3b-2498-4db4-8736-764b7cf44870/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0026beea0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576058030391302 MinTableBarrierTs:449576059603255302 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576058030391302, checkpointTs: 449576058030391302, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576058030391302, checkpointTs: 449576058030391302, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576058030391302, checkpointTs: 449576058030391302, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/931a4a3b-2498-4db4-8736-764b7cf44870 {"id":"931a4a3b-2498-4db4-8736-764b7cf44870","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996562} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e9ec 931a4a3b-2498-4db4-8736-764b7cf44870 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576058030391302,"min-table-barrier-ts":449576059603255302,"admin-job-type":0} + grep -q 'failed to get info:' /tidb/cdc/default/default/task/position/931a4a3b-2498-4db4-8736-764b7cf44870/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0026beea0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576058030391302 MinTableBarrierTs:449576059603255302 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576058030391302, checkpointTs: 449576058030391302, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576058030391302, checkpointTs: 449576058030391302, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576058030391302, checkpointTs: 449576058030391302, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/931a4a3b-2498-4db4-8736-764b7cf44870 {"id":"931a4a3b-2498-4db4-8736-764b7cf44870","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996562} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e9ec 931a4a3b-2498-4db4-8736-764b7cf44870 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576058030391302,"min-table-barrier-ts":449576059603255302,"admin-job-type":0} /tidb/cdc/default/default/task/position/931a4a3b-2498-4db4-8736-764b7cf44870/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' table test.finish_mark not exists for 28-th check, retry later wait process 8420 exit for 2-th time... table processor_delay.t36 exists table processor_delay.t37 not exists for 1-th check, retry later wait process cdc.test exit for 3-th time... Run finished, takes 2.173245338s INSERT - Takes(s): 2.2, Count: 5000, OPS: 2309.5, Avg(us): 3978, Min(us): 1320, Max(us): 67907, 95th(us): 6000, 99th(us): 57000 wait process 8420 exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:56:06 CST 2024] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.87588760.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8420) - No such process wait process 8420 exit for 4-th time... process 8420 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 19:56:06 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.86238625.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/generate_column/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 56-th check, retry later "is-owner": true, run task successfully table test.finish_mark not exists for 29-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:56:07.722564+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e69180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:56:07.725458+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e53180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:56:07.808424+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0009ef500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table processor_delay.t37 exists table processor_delay.t38 not exists for 1-th check, retry later table consistent_replicate_storage_file.usertable exists table consistent_replicate_storage_file.check1 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) owner pid: 4961 owner id 931a4a3b-2498-4db4-8736-764b7cf44870 default owner key /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e9ec [Mon May 6 19:56:08 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.50945096.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 57-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:56:09 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9fe0843e-feee-4226-83aa-f6a304706694 {"id":"9fe0843e-feee-4226-83aa-f6a304706694","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996566} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc21018ed 9fe0843e-feee-4226-83aa-f6a304706694 /tidb/cdc/default/default/changefeed/info/7453ecb3-9c75-45d6-a745-77cf0b3b0da6 {"upstream-id":7365853948448107245,"namespace":"default","changefeed-id":"7453ecb3-9c75-45d6-a745-77cf0b3b0da6","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:55:34.577230405+08:00","start-ts":449576051577716743,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"failed","error":{"time":"2024-05-06T19:56:06.77808868+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576059992276998} /tidb/cdc/default/default/changefeed/status/7453ecb3-9c75-45d6-a745-77cf0b3b0da6 {"checkpoint-ts":449576053753511970,"min-table-barrier-ts":449576053923643423,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365853948448107245 {"id":7365853948448107245,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9fe0843e-feee-4226-83aa-f6a304706694 {"id":"9fe0843e-feee-4226-83aa-f6a304706694","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996566} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc21018ed 9fe0843e-feee-4226-83aa-f6a304706694 /tidb/cdc/default/default/changefeed/info/7453ecb3-9c75-45d6-a745-77cf0b3b0da6 {"upstream-id":7365853948448107245,"namespace":"default","changefeed-id":"7453ecb3-9c75-45d6-a745-77cf0b3b0da6","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:55:34.577230405+08:00","start-ts":449576051577716743,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"failed","error":{"time":"2024-05-06T19:56:06.77808868+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576059992276998} /tidb/cdc/default/default/changefeed/status/7453ecb3-9c75-45d6-a745-77cf0b3b0da6 {"checkpoint-ts":449576053753511970,"min-table-barrier-ts":449576053923643423,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365853948448107245 {"id":7365853948448107245,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9fe0843e-feee-4226-83aa-f6a304706694 {"id":"9fe0843e-feee-4226-83aa-f6a304706694","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996566} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc21018ed 9fe0843e-feee-4226-83aa-f6a304706694 /tidb/cdc/default/default/changefeed/info/7453ecb3-9c75-45d6-a745-77cf0b3b0da6 {"upstream-id":7365853948448107245,"namespace":"default","changefeed-id":"7453ecb3-9c75-45d6-a745-77cf0b3b0da6","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:55:34.577230405+08:00","start-ts":449576051577716743,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"failed","error":{"time":"2024-05-06T19:56:06.77808868+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576059992276998} /tidb/cdc/default/default/changefeed/status/7453ecb3-9c75-45d6-a745-77cf0b3b0da6 {"checkpoint-ts":449576053753511970,"min-table-barrier-ts":449576053923643423,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365853948448107245 {"id":7365853948448107245,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 7453ecb3-9c75-45d6-a745-77cf0b3b0da6 failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=7453ecb3-9c75-45d6-a745-77cf0b3b0da6 + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 7453ecb3-9c75-45d6-a745-77cf0b3b0da6 -s + info='{ "upstream_id": 7365853948448107245, "namespace": "default", "id": "7453ecb3-9c75-45d6-a745-77cf0b3b0da6", "state": "failed", "checkpoint_tso": 449576053753511970, "checkpoint_time": "2024-05-06 19:55:42.944", "error": { "time": "2024-05-06T19:56:06.77808868+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7365853948448107245, "namespace": "default", "id": "7453ecb3-9c75-45d6-a745-77cf0b3b0da6", "state": "failed", "checkpoint_tso": 449576053753511970, "checkpoint_time": "2024-05-06 19:55:42.944", "error": { "time": "2024-05-06T19:56:06.77808868+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7365853948448107245, "namespace": "default", "id": "7453ecb3-9c75-45d6-a745-77cf0b3b0da6", "state": "failed", "checkpoint_tso": 449576053753511970, "checkpoint_time": "2024-05-06 19:55:42.944", "error": { "time": "2024-05-06T19:56:06.77808868+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365853948448107245, '"namespace":' '"default",' '"id":' '"7453ecb3-9c75-45d6-a745-77cf0b3b0da6",' '"state":' '"failed",' '"checkpoint_tso":' 449576053753511970, '"checkpoint_time":' '"2024-05-06' '19:55:42.944",' '"error":' '{' '"time":' '"2024-05-06T19:56:06.77808868+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365853948448107245, '"namespace":' '"default",' '"id":' '"7453ecb3-9c75-45d6-a745-77cf0b3b0da6",' '"state":' '"failed",' '"checkpoint_tso":' 449576053753511970, '"checkpoint_time":' '"2024-05-06' '19:55:42.944",' '"error":' '{' '"time":' '"2024-05-06T19:56:06.77808868+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully [Pipeline] // cache [Pipeline] } table test.finish_mark not exists for 30-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:56:09 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/3c3d0780-7268-4b21-8097-58d5a890cac7 {UpstreamID:7365854083392535616 Namespace:default ID:3c3d0780-7268-4b21-8097-58d5a890cac7 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:56:02.307306078 +0800 CST StartTs:449576058800570371 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b12f30 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576058826522626} {CheckpointTs:449576060530458633 MinTableBarrierTs:449576060530458633 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576060530458633, checkpointTs: 449576060530458633, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b1b7e9fb-33ab-4ac5-af31-dcfe4d4c4e8c {"id":"b1b7e9fb-33ab-4ac5-af31-dcfe4d4c4e8c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996567} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc28a5ca8 b1b7e9fb-33ab-4ac5-af31-dcfe4d4c4e8c /tidb/cdc/default/default/changefeed/info/3c3d0780-7268-4b21-8097-58d5a890cac7 {"upstream-id":7365854083392535616,"namespace":"default","changefeed-id":"3c3d0780-7268-4b21-8097-58d5a890cac7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:56:02.307306078+08:00","start-ts":449576058800570371,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576058826522626} /tidb/cdc/default/default/changefeed/status/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":449576060530458633,"min-table-barrier-ts":449576060530458633,"admin-job-type":0} /tidb/cdc/default/default/task/position/b1b7e9fb-33ab-4ac5-af31-dcfe4d4c4e8c/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854083392535616 {"id":7365854083392535616,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3c3d0780-7268-4b21-8097-58d5a890cac7 {UpstreamID:7365854083392535616 Namespace:default ID:3c3d0780-7268-4b21-8097-58d5a890cac7 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:56:02.307306078 +0800 CST StartTs:449576058800570371 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b12f30 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576058826522626} {CheckpointTs:449576060530458633 MinTableBarrierTs:449576060530458633 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576060530458633, checkpointTs: 449576060530458633, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b1b7e9fb-33ab-4ac5-af31-dcfe4d4c4e8c {"id":"b1b7e9fb-33ab-4ac5-af31-dcfe4d4c4e8c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996567} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc28a5ca8 b1b7e9fb-33ab-4ac5-af31-dcfe4d4c4e8c /tidb/cdc/default/default/changefeed/info/3c3d0780-7268-4b21-8097-58d5a890cac7 {"upstream-id":7365854083392535616,"namespace":"default","changefeed-id":"3c3d0780-7268-4b21-8097-58d5a890cac7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:56:02.307306078+08:00","start-ts":449576058800570371,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576058826522626} /tidb/cdc/default/default/changefeed/status/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":449576060530458633,"min-table-barrier-ts":449576060530458633,"admin-job-type":0} /tidb/cdc/default/default/task/position/b1b7e9fb-33ab-4ac5-af31-dcfe4d4c4e8c/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854083392535616 {"id":7365854083392535616,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":n+ grep -q 'failed to get info:' ull}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3c3d0780-7268-4b21-8097-58d5a890cac7 {UpstreamID:7365854083392535616 Namespace:default ID:3c3d0780-7268-4b21-8097-58d5a890cac7 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:56:02.307306078 +0800 CST StartTs:449576058800570371 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b12f30 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576058826522626} {CheckpointTs:449576060530458633 MinTableBarrierTs:449576060530458633 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576060530458633, checkpointTs: 449576060530458633, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b1b7e9fb-33ab-4ac5-af31-dcfe4d4c4e8c {"id":"b1b7e9fb-33ab-4ac5-af31-dcfe4d4c4e8c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996567} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc28a5ca8 b1b7e9fb-33ab-4ac5-af31-dcfe4d4c4e8c /tidb/cdc/default/default/changefeed/info/3c3d0780-7268-4b21-8097-58d5a890cac7 {"upstream-id":7365854083392535616,"namespace":"default","changefeed-id":"3c3d0780-7268-4b21-8097-58d5a890cac7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:56:02.307306078+08:00","start-ts":449576058800570371,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576058826522626} /tidb/cdc/default/default/changefeed/status/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":449576060530458633,"min-table-barrier-ts":449576060530458633,"admin-job-type":0} /tidb/cdc/default/default/task/position/b1b7e9fb-33ab-4ac5-af31-dcfe4d4c4e8c/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854083392535616 {"id":7365854083392535616,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully table force_replicate_table.t0 exists table force_replicate_table.t1 exists table force_replicate_table.t2 exists table force_replicate_table.t3 not exists for 1-th check, retry later table processor_delay.t38 exists table processor_delay.t39 not exists for 1-th check, retry later [Pipeline] // dir check_changefeed_state http://127.0.0.1:2379 7453ecb3-9c75-45d6-a745-77cf0b3b0da6 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=7453ecb3-9c75-45d6-a745-77cf0b3b0da6 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 7453ecb3-9c75-45d6-a745-77cf0b3b0da6 -s [Pipeline] } table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 4-th check, retry later [Pipeline] // withCredentials + info='{ "upstream_id": 7365853948448107245, "namespace": "default", "id": "7453ecb3-9c75-45d6-a745-77cf0b3b0da6", "state": "normal", "checkpoint_tso": 449576060818030597, "checkpoint_time": "2024-05-06 19:56:09.893", "error": null }' + echo '{ "upstream_id": 7365853948448107245, "namespace": "default", "id": "7453ecb3-9c75-45d6-a745-77cf0b3b0da6", "state": "normal", "checkpoint_tso": 449576060818030597, "checkpoint_time": "2024-05-06 19:56:09.893", "error": null }' { "upstream_id": 7365853948448107245, "namespace": "default", "id": "7453ecb3-9c75-45d6-a745-77cf0b3b0da6", "state": "normal", "checkpoint_tso": 449576060818030597, "checkpoint_time": "2024-05-06 19:56:09.893", "error": null } ++ echo '{' '"upstream_id":' 7365853948448107245, '"namespace":' '"default",' '"id":' '"7453ecb3-9c75-45d6-a745-77cf0b3b0da6",' '"state":' '"normal",' '"checkpoint_tso":' 449576060818030597, '"checkpoint_time":' '"2024-05-06' '19:56:09.893",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365853948448107245, '"namespace":' '"default",' '"id":' '"7453ecb3-9c75-45d6-a745-77cf0b3b0da6",' '"state":' '"normal",' '"checkpoint_tso":' 449576060818030597, '"checkpoint_time":' '"2024-05-06' '19:56:09.893",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully [Pipeline] } table consistent_replicate_storage_file.check1 not exists for 2-th check, retry later [Pipeline] // timeout [Pipeline] } [Pipeline] // stage ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } table mark.finish_mark_3 not exists for 58-th check, retry later table test.finish_mark not exists for 5-th check, retry later table test.finish_mark not exists for 31-th check, retry later table processor_delay.t39 exists table processor_delay.t40 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:56:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc000df53b0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576060927082498 MinTableBarrierTs:449576061189226498 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e02eeda-a428-4cfc-a33d-71e5503c616b {"id":"4e02eeda-a428-4cfc-a33d-71e5503c616b","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996568} /tidb/cdc/default/__cdc_meta__/capture/931a4a3b-2498-4db4-8736-764b7cf44870 {"id":"931a4a3b-2498-4db4-8736-764b7cf44870","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996562} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e9ec 931a4a3b-2498-4db4-8736-764b7cf44870 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077ea3c 4e02eeda-a428-4cfc-a33d-71e5503c616b /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576060927082498,"min-table-barrier-ts":449576061189226498,"admin-job-type":0} /tidb/cdc/default/default/task/position/4e02eeda-a428-4cfc-a33d-71e5503c616b/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/931a4a3b-2498-4db4-8736-764b7cf44870/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc000df53b0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576060927082498 MinTableBarrierTs:449576061189226498 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e02eeda-a428-4cfc-a33d-71e5503c616b {"id":"4e02eeda-a428-4cfc-a33d-71e5503c616b","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996568} /tidb/cdc/default/__cdc_meta__/capture/931a4a3b-2498-4db4-8736-764b7cf44870 {"id":"931a4a3b-2498-4db4-8736-764b7cf44870","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996562} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e9ec 931a4a3b-2498-4db4-8736-764b7cf44870 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077ea3c 4e02eeda-a428-4cfc-a33d-71e5503c616b /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576060927082498,"min-table-barrier-ts":449576061189226498,"admin-job-type":0} /tidb/cdc/default/default/task/position/4e02eeda-a428-4cfc-a33d-71e5503c616b/15d744e6-9+ grep -q 'failed to get info:' 542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/931a4a3b-2498-4db4-8736-764b7cf44870/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc000df53b0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576060927082498 MinTableBarrierTs:449576061189226498 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e02eeda-a428-4cfc-a33d-71e5503c616b {"id":"4e02eeda-a428-4cfc-a33d-71e5503c616b","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996568} /tidb/cdc/default/__cdc_meta__/capture/931a4a3b-2498-4db4-8736-764b7cf44870 {"id":"931a4a3b-2498-4db4-8736-764b7cf44870","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996562} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077e9ec 931a4a3b-2498-4db4-8736-764b7cf44870 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077ea3c 4e02eeda-a428-4cfc-a33d-71e5503c616b /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576060927082498,"min-table-barrier-ts":449576061189226498,"admin-job-type":0} /tidb/cdc/default/default/task/position/4e02eeda-a428-4cfc-a33d-71e5503c616b/15d744e6-9542-4a9e-820f-73b6ebaa1984 + grep -q 'etcd info' {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/931a4a3b-2498-4db4-8736-764b7cf44870/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "931a4a3b-2498-4db4-8736-764b7cf44870 default" | grep id [Pipeline] // node table test1.table1 not exists for 1-th check, retry later table consistent_replicate_storage_file.check1 not exists for 3-th check, retry later [Pipeline] } [Pipeline] // podTemplate [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // withEnv table force_replicate_table.t3 exists table force_replicate_table.t4 not exists for 1-th check, retry later [Pipeline] } [Pipeline] // stage [Pipeline] } table mark.finish_mark_3 not exists for 59-th check, retry later "id": "4e02eeda-a428-4cfc-a33d-71e5503c616b", run task successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.finish_mark not exists for 32-th check, retry later table processor_delay.t40 exists table processor_delay.t41 not exists for 1-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:56:13.723378+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e69180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:56:13.726952+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e53180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:56:13.809796+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0009ef500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":1714996573.9452715,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002a55dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714996573.94534,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714996574.026128,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0041f0380/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714996574.026172,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} table test1.table1 not exists for 2-th check, retry later table test.finish_mark not exists for 6-th check, retry later table force_replicate_table.t4 exists table force_replicate_table.t5 not exists for 1-th check, retry later capture_id: 4e02eeda-a428-4cfc-a33d-71e5503c616b 1 ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '4e02eeda-a428-4cfc-a33d-71e5503c616b' 4e02eeda-a428-4cfc-a33d-71e5503c616b run task successfully curl -X GET http://127.0.0.1:8300/status | grep '"is_owner": false' % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 18037 0 --:--:-- --:--:-- --:--:-- 18200 "is_owner": false run task successfully table consistent_replicate_storage_file.check1 not exists for 4-th check, retry later table mark.finish_mark_3 not exists for 60-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0{"level":"warn","ts":1714996574.8789527,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0016e5500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714996574.8790045,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370bd7880012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:16828, start at 2024-05-06 19:56:13.696287882 +0800 CST m=+5.232741963 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:58:13.702 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:56:13.666 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:46:13.666 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... table test.finish_mark not exists for 33-th check, retry later Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t41 exists table processor_delay.t42 not exists for 1-th check, retry later table test.finish_mark not exists for 7-th check, retry later table test1.table1 exists table test1.table2 not exists for 1-th check, retry later table force_replicate_table.t5 exists table force_replicate_table.t6 not exists for 1-th check, retry later wait process 8628 exit for 1-th time... wait process 8628 exit for 2-th time... table consistent_replicate_storage_file.check1 not exists for 5-th check, retry later wait process 8628 exit for 3-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370bd7880012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:16828, start at 2024-05-06 19:56:13.696287882 +0800 CST m=+5.232741963 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:58:13.702 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:56:13.666 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:46:13.666 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370bd9bc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:16918, start at 2024-05-06 19:56:13.832873375 +0800 CST m=+5.317641052 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:58:13.839 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:56:13.807 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:46:13.807 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8628) - No such process wait process 8628 exit for 4-th time... process 8628 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 19:56:17 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.88518853.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 exists table mark.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 34-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test1.table2 exists nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table force_replicate_table.t6 not exists for 2-th check, retry later check diff successfully table test.finish_mark not exists for 8-th check, retry later table processor_delay.t42 exists table processor_delay.t43 not exists for 1-th check, retry later table consistent_replicate_storage_file.check1 not exists for 6-th check, retry later + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.18289.out cli tso query --pd=http://127.0.0.1:2379 table mark.finish_mark exists table test.finish_mark not exists for 35-th check, retry later check diff successfully nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:56:19.725286+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e69180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:56:19.72779+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e53180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:56:19.810395+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0009ef500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/cli_tls_with_auth/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table force_replicate_table.t6 exists check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 table test.finish_mark not exists for 9-th check, retry later wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... table processor_delay.t43 exists table processor_delay.t44 not exists for 1-th check, retry later table consistent_replicate_storage_file.check1 not exists for 7-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:56:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/3c3d0780-7268-4b21-8097-58d5a890cac7 {UpstreamID:7365854083392535616 Namespace:default ID:3c3d0780-7268-4b21-8097-58d5a890cac7 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:56:02.307306078 +0800 CST StartTs:449576058800570371 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0014b3200 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576058826522626} {CheckpointTs:449576060819079199 MinTableBarrierTs:449576060819079199 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576060819079199, checkpointTs: 449576060819079199, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dadf4707-30d0-4662-8c79-5879d47e80a9 {"id":"dadf4707-30d0-4662-8c79-5879d47e80a9","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996577} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc28a5d31 dadf4707-30d0-4662-8c79-5879d47e80a9 /tidb/cdc/default/default/changefeed/info/3c3d0780-7268-4b21-8097-58d5a890cac7 {"upstream-id":7365854083392535616,"namespace":"default","changefeed-id":"3c3d0780-7268-4b21-8097-58d5a890cac7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:56:02.307306078+08:00","start-ts":449576058800570371,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576058826522626} /tidb/cdc/default/default/changefeed/status/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":449576060819079199,"min-table-barrier-ts":449576060819079199,"admin-job-type":0} /tidb/cdc/default/default/task/position/dadf4707-30d0-4662-8c79-5879d47e80a9/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854083392535616 {"id":7365854083392535616,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3c3d0780-7268-4b21-8097-58d5a890cac7 {UpstreamID:7365854083392535616 Namespace:default ID:3c3d0780-7268-4b21-8097-58d5a890cac7 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:56:02.307306078 +0800 CST StartTs:449576058800570371 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0014b3200 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576058826522626} {CheckpointTs:449576060819079199 MinTableBarrierTs:449576060819079199 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576060819079199, checkpointTs: 449576060819079199, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dadf4707-30d0-4662-8c79-5879d47e80a9 {"id":"dadf4707-30d0-4662-8c79-5879d47e80a9","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996577} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc28a5d31 dadf4707-30d0-4662-8c79-5879d47e80a9 /tidb/cdc/default/default/changefeed/info/3c3d0780-7268-4b21-8097-58d5a890cac7 {"upstream-id":7365854083392535616,"namespace":"default","changefeed-id":"3c3d0780-7268-4b21-8097-58d5a890cac7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:56:02.307306078+08:00","start-ts":449576058800570371,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576058826522626} /tidb/cdc/default/default/changefeed/status/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":449576060819079199,"min-table-barrier-ts":449576060819079199,"admin-job-type":0} /tidb/cdc/default/default/task/position/dadf4707-30d0-4662-8c79-5879d47e80a9/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854083392535616 {"id":7365854083392535616,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3c3d0780-7268-4b21-8097-58d5a890cac7 {UpstreamID:7365854083392535616 Namespace:default ID:3c3d0780-7268-4b21-8097-58d5a890cac7 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:56:02.307306078 +0800 CST StartTs:449576058800570371 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0014b3200 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576058826522626} {CheckpointTs:449576060819079199 MinTableBarrierTs:449576060819079199 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576060819079199, checkpointTs: 449576060819079199, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dadf4707-30d0-4662-8c79-5879d47e80a9 {"id":"dadf4707-30d0-4662-8c79-5879d47e80a9","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996577} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc28a5d31 dadf4707-30d0-4662-8c79-5879d47e80a9 /tidb/cdc/default/default/changefeed/info/3c3d0780-7268-4b21-8097-58d5a890cac7 {"upstream-id":7365854083392535616,"namespace":"default","changefeed-id":"3c3d0780-7268-4b21-8097-58d5a890cac7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:56:02.307306078+08:00","start-ts":449576058800570371,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576058826522626} /tidb/cdc/default/default/changefeed/status/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":449576060819079199,"min-table-barrier-ts":449576060819079199,"admin-job-type":0} /tidb/cdc/default/default/task/position/dadf4707-30d0-4662-8c79-5879d47e80a9/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854083392535616 {"id":7365854083392535616,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully + set +x + tso='449576063235522562 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576063235522562 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449576063235522562 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Mon May 6 19:56:20 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.1832618328.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... run task successfully check_data_subset force_replicate_table.t1 127.0.0.1 4000 127.0.0.1 3306 wait process cdc.test exit for 3-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:56:21 CST 2024] <<<<<< run test case changefeed_resume_with_checkpoint_ts success! >>>>>> run task successfully check_data_subset force_replicate_table.t2 127.0.0.1 4000 127.0.0.1 3306 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:56:21 CST 2024] <<<<<< run test case default_value success! >>>>>> table test.finish_mark not exists for 36-th check, retry later table test.finish_mark not exists for 10-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370c48100006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc, pid:32568, start at 2024-05-06 19:56:20.874457952 +0800 CST m=+5.262873987 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:58:20.881 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:56:20.868 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:46:20.868 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t44 exists table processor_delay.t45 not exists for 1-th check, retry later table consistent_replicate_storage_file.check1 not exists for 8-th check, retry later run task successfully check_data_subset force_replicate_table.t3 127.0.0.1 4000 127.0.0.1 3306 run task successfully check_data_subset force_replicate_table.t4 127.0.0.1 4000 127.0.0.1 3306 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test table test.finish_mark not exists for 37-th check, retry later table test.finish_mark not exists for 11-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370c48100006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc, pid:32568, start at 2024-05-06 19:56:20.874457952 +0800 CST m=+5.262873987 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:58:20.881 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:56:20.868 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:46:20.868 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370c48cc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-cglmc, pid:32656, start at 2024-05-06 19:56:20.938938429 +0800 CST m=+5.277053217 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:58:20.945 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:56:20.915 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:46:20.915 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/generate_column/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/generate_column/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table processor_delay.t45 exists table processor_delay.t46 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:56:23 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/72d88c73-2333-487f-8570-306f6385beb0 {"id":"72d88c73-2333-487f-8570-306f6385beb0","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996580} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc2cf933b 72d88c73-2333-487f-8570-306f6385beb0 /tidb/cdc/default/default/upstream/7365854162068089095 {"id":7365854162068089095,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/72d88c73-2333-487f-8570-306f6385beb0 {"id":"72d88c73-2333-487f-8570-306f6385beb0","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996580} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc2cf933b 72d88c73-2333-487f-8570-306f6385beb0 /tidb/cdc/default/default/upstream/7365854162068089095 {"id":7365854162068089095,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/72d88c73-2333-487f-8570-306f6385beb0 {"id":"72d88c73-2333-487f-8570-306f6385beb0","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996580} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc2cf933b 72d88c73-2333-487f-8570-306f6385beb0 /tidb/cdc/default/default/upstream/7365854162068089095 {"id":7365854162068089095,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449576063235522562 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.18385.out cli changefeed create --start-ts=449576063235522562 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml run task successfully check_data_subset force_replicate_table.t5 127.0.0.1 4000 127.0.0.1 3306 nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365854162068089095,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:56:24.088247521+08:00","start_ts":449576063235522562,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576063235522562,"checkpoint_ts":449576063235522562,"checkpoint_time":"2024-05-06 19:56:19.115"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... table consistent_replicate_storage_file.check1 not exists for 9-th check, retry later wait process cdc.test exit for 1-th time... run task successfully check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 id=19,a=NULL doesn't exist in downstream table force_replicate_table.t6 run task failed 1-th time, retry later wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 3-th time... + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 12087 0 --:--:-- --:--:-- --:--:-- 12277 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 19:56:19.115","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-06 19:56:25.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:56:19.115","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:56:25.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:56:19.115","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:56:25.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-06 19:56:19.115' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:56:19.115","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:56:25.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:56:19.115","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:56:25.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts {"level":"warn","ts":"2024-05-06T19:56:25.72703+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e69180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:56:25.729451+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e53180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} table test.finish_mark not exists for 38-th check, retry later table test.finish_mark not exists for 12-th check, retry later {"level":"warn","ts":"2024-05-06T19:56:25.811088+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0009ef500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + last_synced_ts='1970-01-01 08:00:00.000' table processor_delay.t46 exists + '[' true '!=' true ']' table processor_delay.t47 not exists for 1-th check, retry later + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-06 19:56:25' + echo 'sink_checkpoint_ts is 2024-05-06' 19:56:19.115 sink_checkpoint_ts is 2024-05-06 19:56:19.115 ++ date -d '2024-05-06 19:56:19.115' +%s + checkpoint_timestamp=1714996579 ++ date -d '2024-05-06 19:56:25' +%s + current_timestamp=1714996585 + '[' 6 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table consistent_replicate_storage_file.check1 not exists for 10-th check, retry later check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run task successfully table test.finish_mark not exists for 39-th check, retry later table test.finish_mark not exists for 13-th check, retry later table processor_delay.t47 exists table processor_delay.t48 not exists for 1-th check, retry later table test.t1 exists + sleep 5 empty select id, val from test.availability1 where id=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=1 wait process cdc.test exit for 1-th time... TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.34036.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 19:56:25.318 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column name not equal, one is 5ctmS another is c"] wait process cdc.test exit for 2-th time... table consistent_replicate_storage_file.check1 not exists for 11-th check, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:56:29 CST 2024] <<<<<< run test case force_replicate_table success! >>>>>> table test.finish_mark not exists for 40-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability2 where id=1 and val=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + set +x + tso='449576065674510337 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576065674510337 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:56:29 CST 2024] <<<<<< START cdc server in generate_column case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.3407934081.out server --log-file /tmp/tidb_cdc_test/generate_column/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/generate_column/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 14-th check, retry later table processor_delay.t48 exists table processor_delay.t49 not exists for 1-th check, retry later table consistent_replicate_storage_file.check1 not exists for 12-th check, retry later wait process 8856 exit for 1-th time... wait process 8856 exit for 2-th time... wait process 8856 exit for 3-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8856) - No such process wait process 8856 exit for 4-th time... process 8856 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 19:56:30 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.90479049.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // dir [Pipeline] } table test.finish_mark not exists for 15-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } table test.finish_mark not exists for 41-th check, retry later [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } table processor_delay.t49 exists table processor_delay.t50 not exists for 1-th check, retry later [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } table consistent_replicate_storage_file.check1 not exists for 13-th check, retry later [Pipeline] // withEnv nonempty select id, val from test.availability2 where id=1 and val=22 run task successfully empty select id, val from test.availability2 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [Pipeline] } [Pipeline] // stage [Pipeline] } ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2420 0 --:--:-- --:--:-- --:--:-- 2430 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:56:32.515","puller_resolved_ts":"2024-05-06 19:56:25.715","last_synced_ts":"2024-05-06 19:56:25.765","now_ts":"2024-05-06 19:56:32.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:56:32.515","puller_resolved_ts":"2024-05-06' '19:56:25.715","last_synced_ts":"2024-05-06' '19:56:25.765","now_ts":"2024-05-06' '19:56:32.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:56:32.515","puller_resolved_ts":"2024-05-06' '19:56:25.715","last_synced_ts":"2024-05-06' '19:56:25.765","now_ts":"2024-05-06' '19:56:32.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:56:32 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb991eaa-3a0c-49a6-ac78-3534dd62fbdd {"id":"cb991eaa-3a0c-49a6-ac78-3534dd62fbdd","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996590} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc2e5ab42 cb991eaa-3a0c-49a6-ac78-3534dd62fbdd /tidb/cdc/default/default/upstream/7365854190056975776 {"id":7365854190056975776,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb991eaa-3a0c-49a6-ac78-3534dd62fbdd {"id":"cb991eaa-3a0c-49a6-ac78-3534dd62fbdd","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996590} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc2e5ab42 cb991eaa-3a0c-49a6-ac78-3534dd62fbdd /tidb/cdc/default/default/upstream/7365854190056975776 {"id":7365854190056975776,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb991eaa-3a0c-49a6-ac78-3534dd62fbdd {"id":"cb991eaa-3a0c-49a6-ac78-3534dd62fbdd","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996590} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc2e5ab42 cb991eaa-3a0c-49a6-ac78-3534dd62fbdd /tidb/cdc/default/default/upstream/7365854190056975776 {"id":7365854190056975776,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.34134.out cli changefeed create --start-ts=449576065674510337 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 1119c394-aa4e-4734-ae27-09d90db5d024 Info: {"upstream_id":7365854190056975776,"namespace":"default","id":"1119c394-aa4e-4734-ae27-09d90db5d024","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:56:33.407322588+08:00","start_ts":449576065674510337,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576065674510337,"checkpoint_ts":449576065674510337,"checkpoint_time":"2024-05-06 19:56:28.419"} PASS ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.6% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 42-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:56:33 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/3c3d0780-7268-4b21-8097-58d5a890cac7 {UpstreamID:7365854083392535616 Namespace:default ID:3c3d0780-7268-4b21-8097-58d5a890cac7 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:56:02.307306078 +0800 CST StartTs:449576058800570371 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0032ceb40 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576058826522626} {CheckpointTs:449576060819079199 MinTableBarrierTs:449576060819079199 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576060819079199, checkpointTs: 449576060819079199, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4651c634-c3a3-480f-8f65-e909de313041 {"id":"4651c634-c3a3-480f-8f65-e909de313041","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996591} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc28a5db0 4651c634-c3a3-480f-8f65-e909de313041 /tidb/cdc/default/default/changefeed/info/3c3d0780-7268-4b21-8097-58d5a890cac7 {"upstream-id":7365854083392535616,"namespace":"default","changefeed-id":"3c3d0780-7268-4b21-8097-58d5a890cac7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:56:02.307306078+08:00","start-ts":449576058800570371,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576058826522626} /tidb/cdc/default/default/changefeed/status/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":449576060819079199,"min-table-barrier-ts":449576060819079199,"admin-job-type":0} /tidb/cdc/default/default/task/position/4651c634-c3a3-480f-8f65-e909de313041/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854083392535616 {"id":7365854083392535616,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3c3d0780-7268-4b21-8097-58d5a890cac7 {UpstreamID:7365854083392535616 Namespace:default ID:3c3d0780-7268-4b21-8097-58d5a890cac7 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:56:02.307306078 +0800 CST StartTs:449576058800570371 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0032ceb40 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576058826522626} {CheckpointTs:449576060819079199 MinTableBarrierTs:449576060819079199 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576060819079199, checkpointTs: 449576060819079199, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4651c634-c3a3-480f-8f65-e909de313041 {"id":"4651c634-c3a3-480f-8f65-e909de313041","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996591} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc28a5db0 4651c634-c3a3-480f-8f65-e909de313041 /tidb/cdc/default/default/changefeed/info/3c3d0780-7268-4b21-8097-58d5a890cac7 {"upstream-id":7365854083392535616,"namespace":"default","changefeed-id":"3c3d0780-7268-4b21-8097-58d5a890cac7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:56:02.307306078+08:00","start-ts":449576058800570371,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576058826522626} /tidb/cdc/default/default/changefeed/status/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":449576060819079199,"min-table-barrier-ts":449576060819079199,"admin-job-type":0} /tidb/cdc/default/default/task/position/4651c634-c3a3-480f-8f65-e909de313041/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854083392535616 {"id":7365854083392535616,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3c3d0780-7268-4b21-8097-58d5a890cac7 {UpstreamID:7365854083392535616 Namespace:default ID:3c3d0780-7268-4b21-8097-58d5a890cac7 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:56:02.307306078 +0800 CST StartTs:449576058800570371 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0032ceb40 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576058826522626} {CheckpointTs:449576060819079199 MinTableBarrierTs:449576060819079199 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576060819079199, checkpointTs: 449576060819079199, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4651c634-c3a3-480f-8f65-e909de313041 {"id":"4651c634-c3a3-480f-8f65-e909de313041","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996591} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc28a5db0 4651c634-c3a3-480f-8f65-e909de313041 /tidb/cdc/default/default/changefeed/info/3c3d0780-7268-4b21-8097-58d5a890cac7 {"upstream-id":7365854083392535616,"namespace":"default","changefeed-id":"3c3d0780-7268-4b21-8097-58d5a890cac7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:56:02.307306078+08:00","start-ts":449576058800570371,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576058826522626} /tidb/cdc/default/default/changefeed/status/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":449576060819079199,"min-table-barrier-ts":449576060819079199,"admin-job-type":0} /tidb/cdc/default/default/task/position/4651c634-c3a3-480f-8f65-e909de313041/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854083392535616 {"id":7365854083392535616,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully + run_case_with_unavailable_tikv conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory table processor_delay.t50 exists check diff failed 1-th time, retry later table test.finish_mark exists check diff successfully empty select id, val from test.availability2 where id=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later wait process cdc.test exit for 1-th time... + set +x table consistent_replicate_storage_file.check1 not exists for 14-th check, retry later wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... table generate_column.t not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370d007c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:38258, start at 2024-05-06 19:56:32.681302052 +0800 CST m=+5.438928192 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:58:32.691 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:56:32.671 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:46:32.671 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370d007c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:38258, start at 2024-05-06 19:56:32.681302052 +0800 CST m=+5.438928192 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:58:32.691 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:56:32.671 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:46:32.671 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370d017c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:38342, start at 2024-05-06 19:56:32.766260695 +0800 CST m=+5.439236190 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:58:32.772 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:56:32.735 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:46:32.735 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored wait process cdc.test exit for 3-th time... Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:56:36 CST 2024] <<<<<< run test case many_pk_or_uk success! >>>>>> wait process cdc.test exit for 1-th time... nonempty select id, val from test.availability3 where id=1 and val=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff successfully table consistent_replicate_storage_file.check1 not exists for 15-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 1-th time... table generate_column.t exists table generate_column.t1 not exists for 1-th check, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... wait process cdc.test exit for 3-th time... wait process cdc.test exit for 2-th time... The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting TLS PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:56:37 CST 2024] <<<<<< run test case tidb_mysql_test success! >>>>>> wait process cdc.test exit for 3-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release nonempty select id, val from test.availability3 where id=1 and val=22 run task successfully empty select id, val from test.availability3 where id=1 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:56:38 CST 2024] <<<<<< run test case processor_etcd_worker_delay success! >>>>>> table consistent_replicate_storage_file.check1 not exists for 16-th check, retry later Starting TLS TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table generate_column.t1 exists check diff failed 1-th time, retry later empty select id, val from test.availability3 where id=1 run task successfully delete_owner_key pass table consistent_replicate_storage_file.check1 not exists for 17-th check, retry later Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... Starting TLS TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully \033[0;36m<<< Run all test success >>>\033[0m wait process cdc.test exit for 2-th time... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 3-th time... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) wait process cdc.test exit for 2-th time... [Pipeline] // cache [Pipeline] } cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit run test case test_kill_capture [Mon May 6 19:56:42 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.55595561.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_capture.server1 --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // container [Pipeline] } wait process cdc.test exit for 3-th time... [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } table consistent_replicate_storage_file.check1 exists table consistent_replicate_storage_file.t2 not exists for 1-th check, retry later [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:56:43 CST 2024] <<<<<< run test case generate_column success! >>>>>> wait process 9052 exit for 1-th time... wait process 9052 exit for 2-th time... wait process 9052 exit for 3-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (9052) - No such process wait process 9052 exit for 4-th time... process 9052 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 19:56:43 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.92589260.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file.t2 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:56:45 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00257d200 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576069839978501 MinTableBarrierTs:449576070102122501 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576070102122504, checkpointTs: 449576070102122501, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576070102122502, checkpointTs: 449576070102122501, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576070102122504, checkpointTs: 449576070102122501, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b210f47-0a78-4476-956e-f40519ca9726 {"id":"2b210f47-0a78-4476-956e-f40519ca9726","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996602} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077eb94 2b210f47-0a78-4476-956e-f40519ca9726 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576069839978501,"min-table-barrier-ts":449576070102122501,"admin-job-type":0} /tidb/cdc/default/default/task/position/2b210f47-0a78-4476-956e-f40519ca9726/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00257d200 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576069839978501 MinTableBarrierTs:449576070102122501 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576070102122504, checkpointTs: 449576070102122501, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576070102122502, checkpointTs: 449576070102122501, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576070102122504, checkpointTs: 449576070102122501, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b210f47-0a78-4476-956e-f40519ca9726 {"id":"2b210f47-0a78-4476-956e-f40519ca9726","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996602} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077eb94 2b210f47-0a78-4476-956e-f40519ca9726 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576069839978501,"min-table-barrier-ts":449576070102122501,"admin-job-type":0} /tidb/cdc/default/default/task/position/2b210f47-0a78-4476-956e-f40519ca9726/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00257d200 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576069839978501 MinTableBarrierTs:449576070102122501 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576070102122504, checkpointTs: 449576070102122501, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576070102122502, checkpointTs: 449576070102122501, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576070102122504, checkpointTs: 449576070102122501, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b210f47-0a78-4476-956e-f40519ca9726 {"id":"2b210f47-0a78-4476-956e-f40519ca9726","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996602} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077eb94 2b210f47-0a78-4476-956e-f40519ca9726 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576069839978501,"min-table-barrier-ts":449576070102122501,"admin-job-type":0} /tidb/cdc/default/default/task/position/2b210f47-0a78-4476-956e-f40519ca9726/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370dca800013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:12759, start at 2024-05-06 19:56:45.631161032 +0800 CST m=+5.239547868 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:58:45.641 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:56:45.600 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:46:45.600 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370dcdf00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:21494, start at 2024-05-06 19:56:45.855631288 +0800 CST m=+5.446823412 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:58:45.865 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:56:45.870 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:46:45.870 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:56:44.003 +08:00] [INFO] [dailytest.go:68] ["test pass!!!"] wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Mon May 6 19:56:45 CST 2024] <<<<<< run test case cdc success! >>>>>> table consistent_replicate_storage_file.t2 not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370dd3e00004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:39993, start at 2024-05-06 19:56:46.205787651 +0800 CST m=+5.262428983 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:58:46.212 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:56:46.200 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:46:46.200 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + pd_host=127.0.0.1 + pd_port=2579 + is_tls=true + '[' true == true ']' ++ run_cdc_cli tso query --pd=https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40058.out cli tso query --pd=https://127.0.0.1:2579 "is-owner": true, run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:56:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/3c3d0780-7268-4b21-8097-58d5a890cac7 {UpstreamID:7365854083392535616 Namespace:default ID:3c3d0780-7268-4b21-8097-58d5a890cac7 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:56:02.307306078 +0800 CST StartTs:449576058800570371 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00233a090 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576058826522626} {CheckpointTs:449576060819079199 MinTableBarrierTs:449576060819079199 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576060819079199, checkpointTs: 449576060819079199, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae0f5371-2c20-45ac-a85e-778e00caccc1 {"id":"ae0f5371-2c20-45ac-a85e-778e00caccc1","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996604} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc28a5e24 ae0f5371-2c20-45ac-a85e-778e00caccc1 /tidb/cdc/default/default/changefeed/info/3c3d0780-7268-4b21-8097-58d5a890cac7 {"upstream-id":7365854083392535616,"namespace":"default","changefeed-id":"3c3d0780-7268-4b21-8097-58d5a890cac7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:56:02.307306078+08:00","start-ts":449576058800570371,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576058826522626} /tidb/cdc/default/default/changefeed/status/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":449576060819079199,"min-table-barrier-ts":449576060819079199,"admin-job-type":0} /tidb/cdc/default/default/task/position/ae0f5371-2c20-45ac-a85e-778e00caccc1/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854083392535616 {"id":7365854083392535616,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3c3d0780-7268-4b21-8097-58d5a890cac7 {UpstreamID:7365854083392535616 Namespace:default ID:3c3d0780-7268-4b21-8097-58d5a890cac7 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:56:02.307306078 +0800 CST StartTs:449576058800570371 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00233a090 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576058826522626} {CheckpointTs:449576060819079199 MinTableBarrierTs:449576060819079199 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576060819079199, checkpointTs: 449576060819079199, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae0f5371-2c20-45ac-a85e-778e00caccc1 {"id":"ae0f5371-2c20-45ac-a85e-778e00caccc1","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996604} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc28a5e24 ae0f5371-2c20-45ac-a85e-778e00caccc1 /tidb/cdc/default/default/changefeed/info/3c3d0780-7268-4b21-8097-58d5a890cac7 {"upstream-id":7365854083392535616,"namespace":"default","changefeed-id":"3c3d0780-7268-4b21-8097-58d5a890cac7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:56:02.307306078+08:00","start-ts":449576058800570371,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576058826522626} /tidb/cdc/default/default/changefeed/status/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":449576060819079199,"min-table-barrier-ts":449576060819079199,"admin-job-type":0} /tidb/cdc/default/default/task/position/ae0f5371-2c20-45ac-a85e-778e00caccc1/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854083392535616 {"id":7365854083392535616,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3c3d0780-7268-4b21-8097-58d5a890cac7 {UpstreamID:7365854083392535616 Namespace:default ID:3c3d0780-7268-4b21-8097-58d5a890cac7 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:56:02.307306078 +0800 CST StartTs:449576058800570371 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00233a090 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576058826522626} {CheckpointTs:449576060819079199 MinTableBarrierTs:449576060819079199 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576060819079199, checkpointTs: 449576060819079199, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae0f5371-2c20-45ac-a85e-778e00caccc1 {"id":"ae0f5371-2c20-45ac-a85e-778e00caccc1","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996604} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc28a5e24 ae0f5371-2c20-45ac-a85e-778e00caccc1 /tidb/cdc/default/default/changefeed/info/3c3d0780-7268-4b21-8097-58d5a890cac7 {"upstream-id":7365854083392535616,"namespace":"default","changefeed-id":"3c3d0780-7268-4b21-8097-58d5a890cac7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:56:02.307306078+08:00","start-ts":449576058800570371,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576058826522626} /tidb/cdc/default/default/changefeed/status/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":449576060819079199,"min-table-barrier-ts":449576060819079199,"admin-job-type":0} /tidb/cdc/default/default/task/position/ae0f5371-2c20-45ac-a85e-778e00caccc1/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854083392535616 {"id":7365854083392535616,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_reentrant/run.sh using Sink-Type: mysql... <<================= ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... owner pid: 5564 owner id 2b210f47-0a78-4476-956e-f40519ca9726 table test.availability1 exists VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370dcdf00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:21494, start at 2024-05-06 19:56:45.855631288 +0800 CST m=+5.446823412 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:58:45.865 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:56:45.870 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:46:45.870 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/sink_hang/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370dcde00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:21573, start at 2024-05-06 19:56:45.842558185 +0800 CST m=+5.363352003 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:58:45.849 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:56:45.816 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:46:45.816 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/simple/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/simple/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/simple/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table consistent_replicate_storage_file.t2 not exists for 4-th check, retry later + set +x + tso='449576070663372801 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576070663372801 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370dca800013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:12759, start at 2024-05-06 19:56:45.631161032 +0800 CST m=+5.239547868 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:58:45.641 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:56:45.600 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:46:45.600 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370dca640015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:12850, start at 2024-05-06 19:56:45.6368212 +0800 CST m=+5.171161601 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:58:45.643 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:56:45.643 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:46:45.643 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 19:56:49 CST 2024] <<<<<< START cdc server in cli_tls_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.4010740109.out server --log-file /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_cli_tls_with_auth_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_data_cli_tls_with_auth_tls1 --cluster-id default --config /tmp/tidb_cdc_test/cli_tls_with_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully [Mon May 6 19:56:50 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.57115713.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_storage_file.t2 not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.14221.out cli tso query --pd=http://127.0.0.1:2379 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.22813.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/ddl_reentrant Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/sink_hang Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:56:52 GMT < Content-Length: 1281 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4ac8aed2-22e6-4f1d-a165-aac9236a0b65 {"id":"4ac8aed2-22e6-4f1d-a165-aac9236a0b65","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996609} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dc353b64d 4ac8aed2-22e6-4f1d-a165-aac9236a0b65 /tidb/cdc/default/default/upstream/7365854302341189693 {"id":7365854302341189693,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4ac8aed2-22e6-4f1d-a165-aac9236a0b65 {"id":"4ac8aed2-22e6-4f1d-a165-aac9236a0b65","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996609} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dc353b64d 4ac8aed2-22e6-4f1d-a165-aac9236a0b65 /tidb/cdc/default/default/upstream/7365854302341189693 {"id":7365854302341189693,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4ac8aed2-22e6-4f1d-a165-aac9236a0b65 {"id":"4ac8aed2-22e6-4f1d-a165-aac9236a0b65","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996609} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4dc353b64d 4ac8aed2-22e6-4f1d-a165-aac9236a0b65 /tidb/cdc/default/default/upstream/7365854302341189693 {"id":7365854302341189693,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='449576071625441281 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576071625441281 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table consistent_replicate_storage_file.t2 not exists for 6-th check, retry later + set +x + tso='449576071607091201 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576071607091201 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449576071607091201 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Mon May 6 19:56:52 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1426114263.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 19:56:52 CST 2024] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.2286122863.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:56:53 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001908fc0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576071937130508 MinTableBarrierTs:449576072199012363 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b210f47-0a78-4476-956e-f40519ca9726 {"id":"2b210f47-0a78-4476-956e-f40519ca9726","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996602} /tidb/cdc/default/__cdc_meta__/capture/3d23e8eb-cfc2-4895-96d0-a729b6e6d0db {"id":"3d23e8eb-cfc2-4895-96d0-a729b6e6d0db","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996610} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077eb94 2b210f47-0a78-4476-956e-f40519ca9726 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077ec0c 3d23e8eb-cfc2-4895-96d0-a729b6e6d0db /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576071937130508,"min-table-barrier-ts":449576072199012363,"admin-job-type":0} /tidb/cdc/default/default/task/position/2b210f47-0a78-4476-956e-f40519ca9726/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/3d23e8eb-cfc2-4895-96d0-a729b6e6d0db/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001908fc0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576071937130508 MinTableBarrierTs:449576072199012363 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b210f47-0a78-4476-956e-f40519ca9726 {"id":"2b210f47-0a78-4476-956e-f40519ca9726","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996602} /tidb/cdc/default/__cdc_meta__/capture/3d23e8eb-cfc2-4895-96d0-a729b6e6d0db {"id":"3d23e8eb-cfc2-4895-96d0-a729b6e6d0db","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996610} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077eb94 2b210f47-0a78-4476-956e-f40519ca9726 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077ec0c 3d23e8eb-cfc2-4895-96d0-a729b6e6d0db /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576071937130508,"min-table-barrier-ts":449576072199012363,"admin-job-type":0} /tidb/cdc/default/default/task/position/2b210f47-0a78-4476-956e-f40519ca9726/15d744e6-9+ grep -q 'failed to get info:' 542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/3d23e8eb-cfc2-4895-96d0-a729b6e6d0db/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001908fc0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576071937130508 MinTableBarrierTs:449576072199012363 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b210f47-0a78-4476-956e-f40519ca9726 {"id":"2b210f47-0a78-4476-956e-f40519ca9726","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996602} /tidb/cdc/default/__cdc_meta__/capture/3d23e8eb-cfc2-4895-96d0-a729b6e6d0db {"id":"3d23e8eb-cfc2-4895-96d0-a729b6e6d0db","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996610} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077eb94 2b210f47-0a78-4476-956e-f40519ca9726 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077ec0c 3d23e8eb-cfc2-4895-96d0-a729b6e6d0db /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576071937130508,"min-table-barrier-ts":449576072199012363,"admin-job-type":0} /tidb/cdc/default/default/task/position/2b210f47-0a78-4476-956e-f40519ca9726/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/3d23e8eb-cfc2-4895-96d0-a729b6e6d0db/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "2b210f47-0a78-4476-956e-f40519ca9726" | grep id + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40165.out cli changefeed create --start-ts=449576070663372801 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [WARN] --tz is deprecated in changefeed settings. Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365854302341189693,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:56:55.005976464+08:00","start_ts":449576070663372801,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576070663372801,"checkpoint_ts":449576070663372801,"checkpoint_time":"2024-05-06 19:56:47.450"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release "cluster-id": "default" "id": "3d23e8eb-cfc2-4895-96d0-a729b6e6d0db", "cluster-id": "default" run task successfully table consistent_replicate_storage_file.t2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:56:55 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8bce5b5f-661c-42e9-9626-2574a2e2b680 {"id":"8bce5b5f-661c-42e9-9626-2574a2e2b680","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996612} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc3503b39 8bce5b5f-661c-42e9-9626-2574a2e2b680 /tidb/cdc/default/default/upstream/7365854297480029411 {"id":7365854297480029411,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8bce5b5f-661c-42e9-9626-2574a2e2b680 {"id":"8bce5b5f-661c-42e9-9626-2574a2e2b680","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996612} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc3503b39 8bce5b5f-661c-42e9-9626-2574a2e2b680 /tidb/cdc/default/default/upstream/7365854297480029411 {"id":7365854297480029411,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8bce5b5f-661c-42e9-9626-2574a2e2b680 {"id":"8bce5b5f-661c-42e9-9626-2574a2e2b680","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996612} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc3503b39 8bce5b5f-661c-42e9-9626-2574a2e2b680 /tidb/cdc/default/default/upstream/7365854297480029411 {"id":7365854297480029411,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449576071607091201 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.14320.out cli changefeed create --start-ts=449576071607091201 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365854297480029411,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:56:56.068600476+08:00","start_ts":449576071607091201,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576071607091201,"checkpoint_ts":449576071607091201,"checkpoint_time":"2024-05-06 19:56:51.050"} PASS + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:56:55 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4bb58995-7e07-49dd-9b68-d657bac6fb4d {"id":"4bb58995-7e07-49dd-9b68-d657bac6fb4d","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996613} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc348804f 4bb58995-7e07-49dd-9b68-d657bac6fb4d /tidb/cdc/default/default/upstream/7365854297452147652 {"id":7365854297452147652,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4bb58995-7e07-49dd-9b68-d657bac6fb4d {"id":"4bb58995-7e07-49dd-9b68-d657bac6fb4d","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996613} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc348804f 4bb58995-7e07-49dd-9b68-d657bac6fb4d /tidb/cdc/default/default/upstream/7365854297452147652 {"id":7365854297452147652,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4bb58995-7e07-49dd-9b68-d657bac6fb4d {"id":"4bb58995-7e07-49dd-9b68-d657bac6fb4d","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996613} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc348804f 4bb58995-7e07-49dd-9b68-d657bac6fb4d /tidb/cdc/default/default/upstream/7365854297452147652 {"id":7365854297452147652,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.22923.out cli changefeed create --start-ts=449576071625441281 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ coverage: 2.7% of statements in github.com/pingcap/tiflow/... Create changefeed successfully! ID: ff8f80f3-5922-4a65-9ffa-cb4ecb22cc8f Info: {"upstream_id":7365854297452147652,"namespace":"default","id":"ff8f80f3-5922-4a65-9ffa-cb4ecb22cc8f","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:56:56.372832047+08:00","start_ts":449576071625441281,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576071625441281,"checkpoint_ts":449576071625441281,"checkpoint_time":"2024-05-06 19:56:51.120"} PASS + set +x table test.simple not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later coverage: 2.6% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:56:57 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file.1162211624.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process 9263 exit for 1-th time... wait process 9263 exit for 2-th time... wait process 9263 exit for 3-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (9263) - No such process wait process 9263 exit for 4-th time... process 9263 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 19:56:57 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.94529454.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later + set +x succeed to verify meta placement rules start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.simple exists table test.`simple-dash` not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later table test.t1 exists + sleep 5 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:57:00 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/b7664838-0b70-41d4-9039-099625e43d3e {UpstreamID:7365854079682308468 Namespace:default ID:b7664838-0b70-41d4-9039-099625e43d3e SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:56:02.73421564 +0800 CST StartTs:449576058912768003 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0034415f0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576058938982402} {CheckpointTs:449576073763225607 MinTableBarrierTs:449576073789440002 AdminJobType:noop} span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449576073789440003, checkpointTs: 449576073763225607, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576073789440002, checkpointTs: 449576073763225607, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576073789440002, checkpointTs: 449576073763225607, state: Replicating span: {table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}, resolvedTs: 449576073789440003, checkpointTs: 449576073763225607, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449576073789440003, checkpointTs: 449576073763225607, state: Replicating span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449576073789440002, checkpointTs: 449576073763225607, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449576073763225607, checkpointTs: 449576073763225607, state: Replicating span: {table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}, resolvedTs: 449576073789440002, checkpointTs: 449576073763225607, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e72a978-5aa9-4feb-b4ae-3c599eea7575 {"id":"4e72a978-5aa9-4feb-b4ae-3c599eea7575","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996617} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc2840979 4e72a978-5aa9-4feb-b4ae-3c599eea7575 /tidb/cdc/default/default/changefeed/info/b7664838-0b70-41d4-9039-099625e43d3e {"upstream-id":7365854079682308468,"namespace":"default","changefeed-id":"b7664838-0b70-41d4-9039-099625e43d3e","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:56:02.73421564+08:00","start-ts":449576058912768003,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576058938982402} /tidb/cdc/default/default/changefeed/status/b7664838-0b70-41d4-9039-099625e43d3e {"checkpoint-ts":449576073763225607,"min-table-barrier-ts":449576073789440002,"admin-job-type":0} /tidb/cdc/default/default/task/position/4e72a978-5aa9-4feb-b4ae-3c599eea7575/b7664838-0b70-41d4-9039-099625e43d3e {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854079682308468 {"id":7365854079682308468,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/b7664838-0b70-41d4-9039-099625e43d3e {UpstreamID:7365854079682308468 Namespace:default ID:b7664838-0b70-41d4-9039-099625e43d3e SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:56:02.73421564 +0800 CST StartTs:449576058912768003 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0034415f0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576058938982402} {CheckpointTs:449576073763225607 MinTableBarrierTs:449576073789440002 AdminJobType:noop} span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449576073789440003, checkpointTs: 449576073763225607, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576073789440002, checkpointTs: 449576073763225607, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576073789440002, checkpointTs: 449576073763225607, state: Replicating span: {table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}, resolvedTs: 449576073789440003, checkpointTs: 449576073763225607, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449576073789440003, checkpointTs: 449576073763225607, state: Replicating span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449576073789440002, checkpointTs: 449576073763225607, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449576073763225607, checkpointTs: 449576073763225607, state: Replicating span: {table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}, resolvedTs: 449576073789440002, checkpointTs: 449576073763225607, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e72a978-5aa9-4feb-b4ae-3c599eea7575 {"id":"4e72a978-5aa9-4feb-b4ae-3c599eea7575","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996617} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc2840979 4e72a978-5aa9-4feb-b4ae-3c599eea7575 /tidb/cdc/default/default/changefeed/info/b7664838-0b70-41d4-9039-099625e43d3e {"upstream-id":7365854079682308468,"namespace":"default","changefeed-id":"b7664838-0b70-41d4-9039-099625e43d3e","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:56:02.73421564+08:00","start-ts":449576058912768003,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium+ grep -q 'failed to get info:' -disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576058938982402} /tidb/cdc/default/default/changefeed/status/b7664838-0b70-41d4-9039-099625e43d3e {"checkpoint-ts":449576073763225607,"min-table-barrier-ts":449576073789440002,"admin-job-type":0} /tidb/cdc/default/default/task/position/4e72a978-5aa9-4feb-b4ae-3c599eea7575/b7664838-0b70-41d4-9039-099625e43d3e {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854079682308468 {"id":7365854079682308468,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/b7664838-0b70-41d4-9039-099625e43d3e {UpstreamID:7365854079682308468 Namespace:default ID:b7664838-0b70-41d4-9039-099625e43d3e SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:56:02.73421564 +0800 CST StartTs:449576058912768003 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0034415f0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576058938982402} {CheckpointTs:449576073763225607 MinTableBarrierTs:449576073789440002 AdminJobType:noop} span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449576073789440003, checkpointTs: 449576073763225607, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576073789440002, checkpointTs: 449576073763225607, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576073789440002, checkpointTs: 449576073763225607, state: Replicating span: {table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}, resolvedTs: 449576073789440003, checkpointTs: 449576073763225607, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449576073789440003, checkpointTs: 449576073763225607, state: Replicating span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449576073789440002, checkpointTs: 449576073763225607, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449576073763225607, checkpointTs: 449576073763225607, state: Replicating span: {table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}, resolvedTs: 449576073789440002, checkpointTs: 449576073763225607, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e72a978-5aa9-4feb-b4ae-3c599eea7575 {"id":"4e72a978-5aa9-4feb-b4ae-3c599eea7575","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996617} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc2840979 4e72a978-5aa9-4feb-b4ae-3c599eea7575 /tidb/cdc/default/default/changefeed/info/b7664838-0b70-41d4-9039-099625e43d3e {"upstream-id":7365854079682308468,"namespace":"default","changefeed-id":"b7664838-0b70-41d4-9039-099625e43d3e","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:56:02.73421564+08:00","start-ts":449576058912768003,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium+ grep -q 'etcd info' -disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576058938982402} /tidb/cdc/default/default/changefeed/status/b7664838-0b70-41d4-9039-099625e43d3e {"checkpoint-ts":449576073763225607,"min-table-barrier-ts":449576073789440002,"admin-job-type":0} /tidb/cdc/default/default/task/position/4e72a978-5aa9-4feb-b4ae-3c599eea7575/b7664838-0b70-41d4-9039-099625e43d3e {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854079682308468 {"id":7365854079682308468,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370eab28000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-54cks, pid:20720, start at 2024-05-06 19:56:59.989767011 +0800 CST m=+5.422164967 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:00.001 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:56:59.978 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:46:59.978 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:57:00 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/3c3d0780-7268-4b21-8097-58d5a890cac7 {UpstreamID:7365854083392535616 Namespace:default ID:3c3d0780-7268-4b21-8097-58d5a890cac7 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:56:02.307306078 +0800 CST StartTs:449576058800570371 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002526a20 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576058826522626} {CheckpointTs:449576060819079199 MinTableBarrierTs:449576060845031459 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576060819079199, checkpointTs: 449576060819079199, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576060819079199, checkpointTs: 449576060819079199, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ca594dc-4388-4c52-81e4-634e9386eba4 {"id":"0ca594dc-4388-4c52-81e4-634e9386eba4","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996617} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc28a5eb3 0ca594dc-4388-4c52-81e4-634e9386eba4 /tidb/cdc/default/default/changefeed/info/3c3d0780-7268-4b21-8097-58d5a890cac7 {"upstream-id":7365854083392535616,"namespace":"default","changefeed-id":"3c3d0780-7268-4b21-8097-58d5a890cac7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:56:02.307306078+08:00","start-ts":449576058800570371,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576058826522626} /tidb/cdc/default/default/changefeed/status/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":449576060819079199,"min-table-barrier-ts":449576060845031459,"admin-job-type":0} /tidb/cdc/default/default/task/position/0ca594dc-4388-4c52-81e4-634e9386eba4/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854083392535616 {"id":7365854083392535616,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3c3d0780-7268-4b21-8097-58d5a890cac7 {UpstreamID:7365854083392535616 Namespace:default ID:3c3d0780-7268-4b21-8097-58d5a890cac7 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:56:02.307306078 +0800 CST StartTs:449576058800570371 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002526a20 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576058826522626} {CheckpointTs:449576060819079199 MinTableBarrierTs:449576060845031459 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576060819079199, checkpointTs: 449576060819079199, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576060819079199, checkpointTs: 449576060819079199, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ca594dc-4388-4c52-81e4-634e9386eba4 {"id":"0ca594dc-4388-4c52-81e4-634e9386eba4","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996617} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc28a5eb3 0ca594dc-4388-4c52-81e4-634e9386eba4 /tidb/cdc/default/default/changefeed/info/3c3d0780-7268-4b21-8097-58d5a890cac7 {"upstream-id":7365854083392535616,"namespace":"default","changefeed-id":"3c3d0780-7268-4b21-8097-58d5a890cac7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:56:02.307306078+08:00","start-ts":449576058800570371,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576058826522626} /tidb/cdc/default/default/changefeed/status/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":449576060819079199,"min-table-barrier-ts":449576060845031459,"admin-job-type":0} /tidb/cdc/default/default/task/position/0ca594dc-4388-4c52-81e4-634e9386eba4/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc+ grep -q 'failed to get info:' /default/default/upstream/7365854083392535616 {"id":7365854083392535616,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3c3d0780-7268-4b21-8097-58d5a890cac7 {UpstreamID:7365854083392535616 Namespace:default ID:3c3d0780-7268-4b21-8097-58d5a890cac7 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:56:02.307306078 +0800 CST StartTs:449576058800570371 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002526a20 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576058826522626} {CheckpointTs:449576060819079199 MinTableBarrierTs:449576060845031459 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576060819079199, checkpointTs: 449576060819079199, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576060819079199, checkpointTs: 449576060819079199, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ca594dc-4388-4c52-81e4-634e9386eba4 {"id":"0ca594dc-4388-4c52-81e4-634e9386eba4","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996617} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc28a5eb3 0ca594dc-4388-4c52-81e4-634e9386eba4 /tidb/cdc/default/default/changefeed/info/3c3d0780-7268-4b21-8097-58d5a890cac7 {"upstream-id":7365854083392535616,"namespace":"default","changefeed-id":"3c3d0780-7268-4b21-8097-58d5a890cac7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:56:02.307306078+08:00","start-ts":449576058800570371,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576058826522626} /tidb/cdc/default/default/changefeed/status/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":449576060819079199,"min-table-barrier-ts":449576060845031459,"admin-job-type":0} /tidb/cdc/default/default/task/position/0ca594dc-4388-4c52-81e4-634e9386eba4/3c3d0780-7268-4b21-8097-58d5a890cac7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854083392535616 {"id":7365854083392535616,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully table kill_owner_with_ddl.t1 exists table kill_owner_with_ddl.t2 exists table kill_owner_with_ddl.t3 not exists for 1-th check, retry later table test.`simple-dash` exists + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365854302341189693, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449576071082803211, "checkpoint_time": "2024-05-06 19:56:49.050", "error": null }' + echo '{ "upstream_id": 7365854302341189693, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449576071082803211, "checkpoint_time": "2024-05-06 19:56:49.050", "error": null }' { "upstream_id": 7365854302341189693, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449576071082803211, "checkpoint_time": "2024-05-06 19:56:49.050", "error": null } ++ echo '{' '"upstream_id":' 7365854302341189693, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449576071082803211, '"checkpoint_time":' '"2024-05-06' '19:56:49.050",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365854302341189693, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449576071082803211, '"checkpoint_time":' '"2024-05-06' '19:56:49.050",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) changefeed count 1 check pass, pd_addr: https://127.0.0.1:2579 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40375.out cli changefeed --changefeed-id custom-changefeed-name pause [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table kill_owner_with_ddl.t3 exists PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... check data successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370ec3880013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-b2lwz, pid:7144, start at 2024-05-06 19:57:01.567834514 +0800 CST m=+5.237496206 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:01.574 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:01.538 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:01.538 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370eab28000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-54cks, pid:20720, start at 2024-05-06 19:56:59.989767011 +0800 CST m=+5.422164967 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:00.001 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:56:59.978 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:46:59.978 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370eacac000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-54cks, pid:20797, start at 2024-05-06 19:57:00.08653258 +0800 CST m=+5.460854282 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:00.094 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:00.075 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:00.075 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:57:03 CST 2024] <<<<<< run test case simple success! >>>>>> Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT check diff successfully bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370ec3880013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-b2lwz, pid:7144, start at 2024-05-06 19:57:01.567834514 +0800 CST m=+5.237496206 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:01.574 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:01.538 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:01.538 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370ec5000008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-b2lwz, pid:7215, start at 2024-05-06 19:57:01.639627518 +0800 CST m=+5.238551950 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:01.646 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:01.632 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:01.632 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_reentrant/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_reentrant/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 12244 27.1 0.4 4006880 1931672 ? Sl 19:56 0:07 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv1 jenkins 12245 19.6 0.4 3962340 1882480 ? Sl 19:56 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv2 jenkins 12246 19.8 0.4 3976672 1889004 ? Sl 19:56 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv3 jenkins 12248 26.2 0.4 3996128 1921404 ? Sl 19:56 0:06 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 3188 0 --:--:-- --:--:-- --:--:-- 3197 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:57:03.449","puller_resolved_ts":"2024-05-06 19:56:57.600","last_synced_ts":"2024-05-06 19:56:57.650","now_ts":"2024-05-06 19:57:04.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:57:03.449","puller_resolved_ts":"2024-05-06' '19:56:57.600","last_synced_ts":"2024-05-06' '19:56:57.650","now_ts":"2024-05-06' '19:57:04.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:57:03.449","puller_resolved_ts":"2024-05-06' '19:56:57.600","last_synced_ts":"2024-05-06' '19:56:57.650","now_ts":"2024-05-06' '19:57:04.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370ed0440013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-dwwvr, pid:7809, start at 2024-05-06 19:57:02.397909322 +0800 CST m=+5.266475661 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:02.404 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:02.404 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:02.404 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370ed0440013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-dwwvr, pid:7809, start at 2024-05-06 19:57:02.397909322 +0800 CST m=+5.266475661 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:02.404 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:02.404 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:02.404 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370ed1c00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-dwwvr, pid:7889, start at 2024-05-06 19:57:02.486867899 +0800 CST m=+5.306216944 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:02.497 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:02.498 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:02.498 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_hang/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_hang/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.22072.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:57:06 CST 2024] <<<<<< run test case kill_owner_with_ddl success! >>>>>> skip some DDLs in tidb v4.0.x [Mon May 6 19:57:06 CST 2024] <<<<<< START cdc server in ddl_reentrant case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_reentrant.84588460.out server --log-file /tmp/tidb_cdc_test/ddl_reentrant/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_reentrant/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449576075441733633 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576075441733633 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365854302341189693, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449576074477568013, "checkpoint_time": "2024-05-06 19:57:02.000", "error": null }' + echo '{ "upstream_id": 7365854302341189693, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449576074477568013, "checkpoint_time": "2024-05-06 19:57:02.000", "error": null }' { "upstream_id": 7365854302341189693, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449576074477568013, "checkpoint_time": "2024-05-06 19:57:02.000", "error": null } ++ echo '{' '"upstream_id":' 7365854302341189693, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449576074477568013, '"checkpoint_time":' '"2024-05-06' '19:57:02.000",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365854302341189693, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449576074477568013, '"checkpoint_time":' '"2024-05-06' '19:57:02.000",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40473.out cli changefeed update --pd=https://127.0.0.1:2579 --config=/tmp/tidb_cdc_test/cli_tls_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name [Mon May 6 19:57:07 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.2211522117.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 19:57:07 CST 2024] <<<<<< START cdc server in sink_hang case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=2*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_hang.91399141.out server --log-file /tmp/tidb_cdc_test/sink_hang/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_hang/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7365854302341189693,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:56:55.005976464+08:00","start_ts":449576070663372801,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":0,"checkpoint_ts":449576074477568013,"checkpoint_time":"2024-05-06 19:57:02.000"} PASS coverage: 3.0% of statements in github.com/pingcap/tiflow/... nonempty select id, val from test.availability1 where id=2 and val=2 run task successfully wait process cdc.test exit for 1-th time... + set +x wait process cdc.test exit for 2-th time... + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40512.out cli changefeed --changefeed-id custom-changefeed-name resume + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:57:09 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2968715-1004-4c31-8424-525661a651a7 {"id":"d2968715-1004-4c31-8424-525661a651a7","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996627} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc386ae34 d2968715-1004-4c31-8424-525661a651a7 /tidb/cdc/default/default/upstream/7365854368045912584 {"id":7365854368045912584,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2968715-1004-4c31-8424-525661a651a7 {"id":"d2968715-1004-4c31-8424-525661a651a7","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996627} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc386ae34 d2968715-1004-4c31-8424-525661a651a7 /tidb/cdc/default/default/upstream/7365854368045912584 {"id":7365854368045912584,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2968715-1004-4c31-8424-525661a651a7 {"id":"d2968715-1004-4c31-8424-525661a651a7","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996627} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc386ae34 d2968715-1004-4c31-8424-525661a651a7 /tidb/cdc/default/default/upstream/7365854368045912584 {"id":7365854368045912584,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ------------------------------------------ test ddl create database ddl_reentrant, is_reentrant: false restored_sql: CREATE DATABASE `ddl_reentrant` check_ts_forward 1dc3442d-acde-443c-a8dd-b67ebf61b0bf VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370f36a00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3n4n5, pid:23803, start at 2024-05-06 19:57:08.942852199 +0800 CST m=+5.248480764 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:08.953 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:08.954 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:08.954 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:57:10 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9fa748e-7c5c-4f4b-946d-3c53c89c7e67 {"id":"b9fa748e-7c5c-4f4b-946d-3c53c89c7e67","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996627} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc3879c51 b9fa748e-7c5c-4f4b-946d-3c53c89c7e67 /tidb/cdc/default/default/upstream/7365854360787253570 {"id":7365854360787253570,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9fa748e-7c5c-4f4b-946d-3c53c89c7e67 {"id":"b9fa748e-7c5c-4f4b-946d-3c53c89c7e67","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996627} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc3879c51 b9fa748e-7c5c-4f4b-946d-3c53c89c7e67 /tidb/cdc/default/default/upstream/7365854360787253570 {"id":7365854360787253570,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9fa748e-7c5c-4f4b-946d-3c53c89c7e67 {"id":"b9fa748e-7c5c-4f4b-946d-3c53c89c7e67","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996627} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc3879c51 b9fa748e-7c5c-4f4b-946d-3c53c89c7e67 /tidb/cdc/default/default/upstream/7365854360787253570 {"id":7365854360787253570,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x PASS coverage: 2.3% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:57:10 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/523fd1d2-ca54-49ee-bbad-0bb2556c4abe {"id":"523fd1d2-ca54-49ee-bbad-0bb2556c4abe","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996627} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc389aa36 523fd1d2-ca54-49ee-bbad-0bb2556c4abe /tidb/cdc/default/default/upstream/7365854362783211499 {"id":7365854362783211499,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/523fd1d2-ca54-49ee-bbad-0bb2556c4abe {"id":"523fd1d2-ca54-49ee-bbad-0bb2556c4abe","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996627} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc389aa36 523fd1d2-ca54-49ee-bbad-0bb2556c4abe /tidb/cdc/default/default/upstream/7365854362783211499 {"id":7365854362783211499,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/523fd1d2-ca54-49ee-bbad-0bb2556c4abe {"id":"523fd1d2-ca54-49ee-bbad-0bb2556c4abe","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996627} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc389aa36 523fd1d2-ca54-49ee-bbad-0bb2556c4abe /tidb/cdc/default/default/upstream/7365854362783211499 {"id":7365854362783211499,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit run test case test_hang_up_capture [Mon May 6 19:57:10 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.59195921.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 lease 22318f4dc3879c51 revoked + set +x changefeed is working normally rts: 449576076532252700->449576076938838000 checkpoint: 449576076532252700->449576076938838000 run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370f36a00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3n4n5, pid:23803, start at 2024-05-06 19:57:08.942852199 +0800 CST m=+5.248480764 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:08.953 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:08.954 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:08.954 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d370f374c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3n4n5, pid:23881, start at 2024-05-06 19:57:08.963126616 +0800 CST m=+5.194187341 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:08.972 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:08.947 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:08.947 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/split_region/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 19:57:04.025 +08:00] [INFO] [main.go:86] ["running ddl test: 0 createDropSchemaDDL"] [2024/05/06 19:57:04.207 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 19:57:04.221 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 19:57:04.674 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 19:57:04.676 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 19:57:04.993 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 19:57:04.994 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 19:57:05.130 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 19:57:05.467 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/06 19:57:05.591 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 19:57:05.596 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 19:57:05.924 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 19:57:05.927 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 19:57:06.048 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/06 19:57:06.376 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 19:57:06.504 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 19:57:06.510 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/06 19:57:06.836 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 19:57:06.843 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 19:57:06.974 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 19:57:07.292 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 19:57:07.453 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 19:57:07.463 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 19:57:07.465 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 19:57:07.755 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 19:57:07.925 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 19:57:08.214 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/06 19:57:08.676 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 19:57:08.682 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/06 19:57:08.690 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 19:57:08.704 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [mysql] 2024/05/06 19:57:09 connection.go:299: invalid connection check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/06 19:57:11.579 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="CREATE DATABASE `ddl_reentrant`"] [duration=77.389809ms] [namespace=default] [changefeed=1dc3442d-acde-443c-a8dd-b67ebf61b0bf] run task successfully check_changefeed_status 127.0.0.1:8300 1caf468b-6141-4996-90e8-54c4758c2e76 normal last_error null + endpoint=127.0.0.1:8300 + changefeed_id=1caf468b-6141-4996-90e8-54c4758c2e76 + expected_state=normal + field=last_error + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/1caf468b-6141-4996-90e8-54c4758c2e76/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1184 0 --:--:-- --:--:-- --:--:-- 1194 + info='{"state":"normal","resolved_ts":449576076916555800,"checkpoint_ts":449576076772114461}' + echo '{"state":"normal","resolved_ts":449576076916555800,"checkpoint_ts":449576076772114461}' {"state":"normal","resolved_ts":449576076916555800,"checkpoint_ts":449576076772114461} ++ echo '{"state":"normal","resolved_ts":449576076916555800,"checkpoint_ts":449576076772114461}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_error ]] ++ echo '{"state":"normal","resolved_ts":449576076916555800,"checkpoint_ts":449576076772114461}' ++ jq -r .last_error.message + error_msg=null + [[ ! null =~ null ]] run task successfully check_changefeed_status 127.0.0.1:8300 1caf468b-6141-4996-90e8-54c4758c2e76 normal last_warning null + endpoint=127.0.0.1:8300 + changefeed_id=1caf468b-6141-4996-90e8-54c4758c2e76 + expected_state=normal + field=last_warning + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/1caf468b-6141-4996-90e8-54c4758c2e76/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1049 0 --:--:-- --:--:-- --:--:-- 1061 + info='{"state":"normal","resolved_ts":449576076916555800,"checkpoint_ts":449576076916555800}' + echo '{"state":"normal","resolved_ts":449576076916555800,"checkpoint_ts":449576076916555800}' {"state":"normal","resolved_ts":449576076916555800,"checkpoint_ts":449576076916555800} ++ echo '{"state":"normal","resolved_ts":449576076916555800,"checkpoint_ts":449576076916555800}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_warning ]] ++ echo '{"state":"normal","resolved_ts":449576076916555800,"checkpoint_ts":449576076916555800}' ++ jq -r .last_warning.message + error_msg=null + [[ ! null =~ null ]] run task successfully table sink_hang.t1 exists table sink_hang.t2 not exists for 1-th check, retry later table capture_session_done_during_task.t exists check diff failed 1-th time, retry later resume changefeed 1dc3442d-acde-443c-a8dd-b67ebf61b0bf from 449576076571574283 check_ts_forward 1dc3442d-acde-443c-a8dd-b67ebf61b0bf =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.25150.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:57:13 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00137b440 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576077455261698 MinTableBarrierTs:449576077455261698 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576077455261701, checkpointTs: 449576077455261698, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576077455261699, checkpointTs: 449576077455261698, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576077455261701, checkpointTs: 449576077455261698, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/62c96b85-9f6e-4919-96e3-b2d125eede78 {"id":"62c96b85-9f6e-4919-96e3-b2d125eede78","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996631} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077ecba 62c96b85-9f6e-4919-96e3-b2d125eede78 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576077455261698,"min-table-barrier-ts":449576077455261698,"admin-job-type":0} /tidb/cdc/default/default/task/position/62c96b85-9f6e-4919-96e3-b2d125eede78/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00137b440 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576077455261698 MinTableBarrierTs:449576077455261698 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576077455261701, checkpointTs: 449576077455261698, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576077455261699, checkpointTs: 449576077455261698, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576077455261701, checkpointTs: 449576077455261698, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/62c96b85-9f6e-4919-96e3-b2d125eede78 {"id":"62c96b85-9f6e-4919-96e3-b2d125eede78","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996631} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077ecba 62c96b85-9f6e-4919-96e3-b2d125eede78 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576077455261698,"min-table-barrier-ts":449576077455261698,"admin-job-type":0} /tidb/cdc/default/default/task/position/62c96b85-9f6e-4919-96e3-b2d125eede78/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00137b440 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576077455261698 MinTableBarrierTs:449576077455261698 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576077455261701, checkpointTs: 449576077455261698, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576077455261699, checkpointTs: 449576077455261698, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576077455261701, checkpointTs: 449576077455261698, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/62c96b85-9f6e-4919-96e3-b2d125eede78 {"id":"62c96b85-9f6e-4919-96e3-b2d125eede78","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996631} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077ecba 62c96b85-9f6e-4919-96e3-b2d125eede78 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576077455261698,"min-table-barrier-ts":449576077455261698,"admin-job-type":0} /tidb/cdc/default/default/task/position/62c96b85-9f6e-4919-96e3-b2d125eede78/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'etcd info' /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' [2024/05/06 19:57:14.106 +08:00] [INFO] [main.go:86] ["running ddl test: 1 truncateDDL"] [2024/05/06 19:57:14.313 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 19:57:14.321 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 19:57:14.492 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 19:57:14.493 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 19:57:14.513 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 19:57:14.515 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 19:57:14.662 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/06 19:57:14.691 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 19:57:14.825 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 19:57:14.826 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 19:57:14.865 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 19:57:14.875 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 19:57:15.043 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] table sink_hang.t2 exists check diff failed 1-th time, retry later + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365854302341189693, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449576077636403202, "checkpoint_time": "2024-05-06 19:57:14.050", "error": null }' + echo '{ "upstream_id": 7365854302341189693, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449576077636403202, "checkpoint_time": "2024-05-06 19:57:14.050", "error": null }' { "upstream_id": 7365854302341189693, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449576077636403202, "checkpoint_time": "2024-05-06 19:57:14.050", "error": null } ++ echo '{' '"upstream_id":' 7365854302341189693, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449576077636403202, '"checkpoint_time":' '"2024-05-06' '19:57:14.050",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365854302341189693, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449576077636403202, '"checkpoint_time":' '"2024-05-06' '19:57:14.050",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40602.out cli changefeed --changefeed-id custom-changefeed-name remove check diff failed 2-th time, retry later changefeed is working normally rts: 449576076571574300->449576077725007900 checkpoint: 449576076571574300->449576077725007900 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2024/05/06 19:57:13.509 +08:00] [INFO] [mysql_ddl_sink.go:125] ["Execute DDL failed, but error can be ignored"] [startTs=449576076571574274] [ddl="CREATE DATABASE `ddl_reentrant`"] [namespace=default] [changefeed=1dc3442d-acde-443c-a8dd-b67ebf61b0bf] [error="Error 1007 (HY000): Can't create database 'ddl_reentrant'; database exists"] run task successfully ------------------------------------------ test ddl create table ddl_reentrant.t3 (a int primary key, b int) partition by range(a) (partition p0 values less than (1000), partition p1 values less than (2000)), is_reentrant: false restored_sql: CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000)) check_ts_forward 1dc3442d-acde-443c-a8dd-b67ebf61b0bf "is-owner": true, run task successfully + set +x + tso='449576077716357121 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576077716357121 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:57:16 CST 2024] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.2519225194.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 449576077898547202 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/ PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/owner_remove_table_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... owner pid: 5924 owner id 62c96b85-9f6e-4919-96e3-b2d125eede78 [Mon May 6 19:57:16 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.60476049.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:57:16.499 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 19:57:16.603 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 19:57:16.620 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/06 19:57:16.681 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 19:57:16.683 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 19:57:16.888 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 19:57:16.889 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... check diff failed 2-th time, retry later [2024/05/06 19:57:17.065 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 19:57:17.069 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 19:57:17.071 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 19:57:17.089 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] check diff successfully check diff failed 1-th time, retry later changefeed is working normally rts: 449576077725007900->449576077908508700 checkpoint: 449576077908508700->449576077908508700 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/06 19:57:15.536 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000))"] [duration=84.383912ms] [namespace=default] [changefeed=1dc3442d-acde-443c-a8dd-b67ebf61b0bf] run task successfully [2024/05/06 19:57:17.252 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/06 19:57:17.254 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 19:57:17.421 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 19:57:17.430 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/06 19:57:17.437 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 19:57:17.460 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] + set +x [2024/05/06 19:57:17.595 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/06 19:57:17.619 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/06 19:57:17.770 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 19:57:17.787 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/06 19:57:17.795 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 19:57:17.815 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/06 19:57:17.970 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] resume changefeed 1dc3442d-acde-443c-a8dd-b67ebf61b0bf from 449576077908508695 check_ts_forward 1dc3442d-acde-443c-a8dd-b67ebf61b0bf [2024/05/06 19:57:18.034 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/06 19:57:18.153 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 19:57:18.180 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/06 19:57:18.211 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 19:57:18.338 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] Verifying downstream PD is started... [2024/05/06 19:57:18.517 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/06 19:57:18.543 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/06 19:57:18.691 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] start tidb cluster in /tmp/tidb_cdc_test/owner_remove_table_error Starting Upstream PD... check diff failed 3-th time, retry later Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:57:19 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/76b4b2b9-a2ad-4986-ab3c-7ee7f277d0ae {"id":"76b4b2b9-a2ad-4986-ab3c-7ee7f277d0ae","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996636} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc3a37356 76b4b2b9-a2ad-4986-ab3c-7ee7f277d0ae /tidb/cdc/default/default/upstream/7365854397621629393 {"id":7365854397621629393,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/76b4b2b9-a2ad-4986-ab3c-7ee7f277d0ae {"id":"76b4b2b9-a2ad-4986-ab3c-7ee7f277d0ae","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996636} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc3a37356 76b4b2b9-a2ad-4986-ab3c-7ee7f277d0ae /tidb/cdc/default/default/upstream/7365854397621629393 {"id":7365854397621629393,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/76b4b2b9-a2ad-4986-ab3c-7ee7f277d0ae {"id":"76b4b2b9-a2ad-4986-ab3c-7ee7f277d0ae","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996636} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc3a37356 76b4b2b9-a2ad-4986-ab3c-7ee7f277d0ae /tidb/cdc/default/default/upstream/7365854397621629393 {"id":7365854397621629393,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.25250.out cli changefeed create --start-ts=449576077716357121 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config /tmp/tidb_cdc_test/split_region/pulsar_test.toml Create changefeed successfully! ID: 3c5ec77a-68b8-472a-82e4-562849855bf4 Info: {"upstream_id":7365854397621629393,"namespace":"default","id":"3c5ec77a-68b8-472a-82e4-562849855bf4","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:57:19.601468082+08:00","start_ts":449576077716357121,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576077716357121,"checkpoint_ts":449576077716357121,"checkpoint_time":"2024-05-06 19:57:14.355"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... check diff successfully wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:57:19 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0019a19e0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576079028125701 MinTableBarrierTs:449576079028125701 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/62c96b85-9f6e-4919-96e3-b2d125eede78 {"id":"62c96b85-9f6e-4919-96e3-b2d125eede78","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996631} /tidb/cdc/default/__cdc_meta__/capture/fbd4e753-9975-43fa-942f-ed61c34e5dbd {"id":"fbd4e753-9975-43fa-942f-ed61c34e5dbd","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996637} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077ecba 62c96b85-9f6e-4919-96e3-b2d125eede78 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077ed12 fbd4e753-9975-43fa-942f-ed61c34e5dbd /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576079028125701,"min-table-barrier-ts":449576079028125701,"admin-job-type":0} /tidb/cdc/default/default/task/position/62c96b85-9f6e-4919-96e3-b2d125eede78/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/fbd4e753-9975-43fa-942f-ed61c34e5dbd/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0019a19e0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576079028125701 MinTableBarrierTs:449576079028125701 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/62c96b85-9f6e-4919-96e3-b2d125eede78 {"id":"62c96b85-9f6e-4919-96e3-b2d125eede78","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996631} /tidb/cdc/default/__cdc_meta__/capture/fbd4e753-9975-43fa-942f-ed61c34e5dbd {"id":"fbd4e753-9975-43fa-942f-ed61c34e5dbd","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996637} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077ecba 62c96b85-9f6e-4919-96e3-b2d125eede78 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077ed12 fbd4e753-9975-43fa-942f-ed61c34e5dbd /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576079028125701,"min-table-barrier-ts":449576079028125701,"admin-job-type":0} /tidb/cdc/default/default/task/position/62c96b85-9f6e-4919-96e3-b2d125eede78/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/fbd4e753-9975-43fa-942f-ed61c34e5dbd/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0019a19e0 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576079028125701 MinTableBarrierTs:449576079028125701 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/62c96b85-9f6e-4919-96e3-b2d125eede78 {"id":"62c96b85-9f6e-4919-96e3-b2d125eede78","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996631} /tidb/cdc/default/__cdc_meta__/capture/fbd4e753-9975-43fa-942f-ed61c34e5dbd {"id":"fbd4e753-9975-43fa-942f-ed61c34e5dbd","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996637} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077ecba 62c96b85-9f6e-4919-96e3-b2d125eede78 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077ed12 fbd4e753-9975-43fa-942f-ed61c34e5dbd /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576079028125701,"min-table-barrier-ts":449576079028125701,"admin-job-type":0} /tidb/cdc/default/default/task/position/62c96b85-9f6e-4919-96e3-b2d125eede78/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/fbd4e753-9975-43fa-942f-ed61c34e5dbd/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "62c96b85-9f6e-4919-96e3-b2d125eede78" | grep id run task failed 1-th time, retry later wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... wait process cdc.test exit for 3-th time... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release changefeed count 0 check pass, pd_addr: https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40692.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x table split_region.test1 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:57:21 CST 2024] <<<<<< run test case capture_session_done_during_task success! >>>>>> [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365854302341189693,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:57:21.403377603+08:00","start_ts":449576079510732802,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576079510732802,"checkpoint_ts":449576079510732802,"checkpoint_time":"2024-05-06 19:57:21.200"} PASS "cluster-id": "default" "id": "fbd4e753-9975-43fa-942f-ed61c34e5dbd", "cluster-id": "default" run task successfully check diff failed 4-th time, retry later coverage: 2.6% of statements in github.com/pingcap/tiflow/... check_ts_forward 1dc3442d-acde-443c-a8dd-b67ebf61b0bf check_redo_resolved_ts b7664838-0b70-41d4-9039-099625e43d3e 449576079818752002 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/b7664838-0b70-41d4-9039-099625e43d3e/meta global resolved ts 449576074025369601 not forward to 449576079818752002 run task failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=3 and val=3 Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... + set +x TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later table split_region.test1 exists table split_region.test2 not exists for 1-th check, retry later changefeed is working normally rts: 449576079560015900->449576080084303900 checkpoint: 449576079560015900->449576080084303900 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2024/05/06 19:57:19.552 +08:00] [INFO] [mysql_ddl_sink.go:125] ["Execute DDL failed, but error can be ignored"] [startTs=449576077908508688] [ddl="CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000))"] [namespace=default] [changefeed=1dc3442d-acde-443c-a8dd-b67ebf61b0bf] [error="Error 1050 (42S01): Table 'ddl_reentrant.t3' already exists"] run task successfully ------------------------------------------ test ddl create table ddl_reentrant.t4 (a int primary key, b int), is_reentrant: false restored_sql: CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT) check_ts_forward 1dc3442d-acde-443c-a8dd-b67ebf61b0bf check_redo_resolved_ts b7664838-0b70-41d4-9039-099625e43d3e 449576079818752002 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/b7664838-0b70-41d4-9039-099625e43d3e/meta run task successfully nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:57:24.550 +08:00] [INFO] [main.go:86] ["running ddl test: 2 addDropColumnDDL"] [2024/05/06 19:57:24.740 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 19:57:24.748 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 19:57:24.907 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 19:57:24.908 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 19:57:24.914 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 19:57:24.916 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 19:57:25.072 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 19:57:25.080 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/06 19:57:25.227 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 19:57:25.228 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 19:57:25.244 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 19:57:25.245 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... check diff failed 6-th time, retry later [2024/05/06 19:57:25.393 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/06 19:57:25.413 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] table split_region.test2 exists check diff successfully [2024/05/06 19:57:25.565 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/06 19:57:25.565 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 19:57:25.583 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 19:57:25.585 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 19:57:25.732 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 19:57:25.744 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] wait process cdc.test exit for 2-th time... [2024/05/06 19:57:25.921 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 19:57:25.922 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 19:57:25.934 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 19:57:25.935 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 19:57:26.096 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 19:57:26.104 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365854302341189693, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449576080519987203, "checkpoint_time": "2024-05-06 19:57:25.050", "error": null }' + echo '{ "upstream_id": 7365854302341189693, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449576080519987203, "checkpoint_time": "2024-05-06 19:57:25.050", "error": null }' { "upstream_id": 7365854302341189693, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449576080519987203, "checkpoint_time": "2024-05-06 19:57:25.050", "error": null } ++ echo '{' '"upstream_id":' 7365854302341189693, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449576080519987203, '"checkpoint_time":' '"2024-05-06' '19:57:25.050",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365854302341189693, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449576080519987203, '"checkpoint_time":' '"2024-05-06' '19:57:25.050",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] [2024/05/06 19:57:26.266 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/06 19:57:26.267 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 19:57:26.275 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 19:57:26.277 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/06 19:57:26.431 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/06 19:57:26.442 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] wait process cdc.test exit for 3-th time... [2024/05/06 19:57:26.594 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/06 19:57:26.595 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 19:57:26.606 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 19:57:26.608 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/06 19:57:26.762 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/06 19:57:26.766 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:57:26.928 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 19:57:26.930 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/06 19:57:26.932 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/06 19:57:26.933 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 19:57:26.843 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 19:57:26.844 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2024/05/06 19:57:26.844 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=:6060] [2024/05/06 19:57:26.844 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 19:57:26.845 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449576074169548813] [resolvedTs=449576080080896003] [2024/05/06 19:57:26.849 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 19:57:26.849 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 19:57:26.849 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/06 19:57:26.850 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=5.060603ms] [2024/05/06 19:57:26.854 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 19:57:26.857 +08:00] [INFO] [mysql.go:197] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 19:57:26.858 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:57:26.858 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:57:26.858 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:57:26.858 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:57:26.858 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:57:26.858 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:57:26.858 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:57:26.858 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:57:26.858 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:57:26.858 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:57:26.858 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:57:26.858 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:57:26.858 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:57:26.858 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:57:26.858 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:57:26.858 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:57:26.859 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 19:57:26.859 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 19:57:26.859 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 19:57:26.859 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 19:57:26.859 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 19:57:26.859 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 19:57:26.859 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 19:57:26.859 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 19:57:26.859 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 19:57:26.859 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 19:57:26.859 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 19:57:26.859 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 19:57:26.859 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 19:57:26.859 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 19:57:26.859 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 19:57:26.859 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 19:57:26.859 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 19:57:26.859 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 19:57:26.859 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/06 19:57:26.863 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 19:57:26.866 +08:00] [INFO] [mysql_ddl_sink.go:105] ["MySQL DDL sink is created"] [namespace=] [changefeed=] changefeed is working normally rts: 449576080084303900->449576080608591900 checkpoint: 449576080084303900->449576080320233500 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/06 19:57:25.682 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT)"] [duration=80.591408ms] [namespace=default] [changefeed=1dc3442d-acde-443c-a8dd-b67ebf61b0bf] run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40828.out cli unsafe delete-service-gc-safepoint [2024/05/06 19:57:27.085 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/06 19:57:27.101 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/06 19:57:27.241 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/06 19:57:27.243 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/06 19:57:27.267 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/06 19:57:27.268 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] resume changefeed 1dc3442d-acde-443c-a8dd-b67ebf61b0bf from 449576080320233489 check_ts_forward 1dc3442d-acde-443c-a8dd-b67ebf61b0bf Confirm that you know what this command will do and use it at your own risk [Y/N] [2024/05/06 19:57:27.410 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] CDC service GC safepoint truncated in PD! [2024/05/06 19:57:27.450 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:57:27.385 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=540.720269ms] [2024/05/06 19:57:27.386 +08:00] [ERROR] [redo.go:268] ["ignore unsupported DDL"] [ddl="{\"StartTs\":449576074169548807,\"CommitTs\":449576074169548813,\"Query\":\"ALTER TABLE `consistent_replicate_storage_file`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_file`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_storage_file\",\"Table\":\"t1\",\"TableID\":110,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 19:57:27.584 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/06 19:57:27.586 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/06 19:57:27.630 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/06 19:57:27.631 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/06 19:57:27.749 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/06 19:57:27.818 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] check diff failed 7-th time, retry later [2024/05/06 19:57:27.928 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/06 19:57:27.930 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/06 19:57:27.995 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/06 19:57:27.996 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/06 19:57:28.092 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/06 19:57:28.175 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/06 19:57:28.272 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/06 19:57:28.274 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/06 19:57:28.355 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/06 19:57:28.356 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/06 19:57:28.449 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/06 19:57:28.532 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/06 19:57:28.618 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/06 19:57:28.620 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/06 19:57:28.745 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/05/06 19:57:28.746 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40861.out cli unsafe reset --no-confirm --pd=https://127.0.0.1:2579 [2024/05/06 19:57:28.576 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}] [checkpointTs=449576080080896003] [2024/05/06 19:57:28.576 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}] [checkpointTs=449576080080896003] [2024/05/06 19:57:28.676 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}] [checkpointTs=449576080080896003] [2024/05/06 19:57:28.676 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}] [checkpointTs=449576080080896003] [2024/05/06 19:57:28.835 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/05/06 19:57:28.955 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/05/06 19:57:29.021 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/05/06 19:57:29.023 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later run task failed 1-th time, retry later [2024/05/06 19:57:29.122 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] [2024/05/06 19:57:29.123 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/05/06 19:57:29.191 +08:00] [INFO] [main.go:220] ["0 insert success: 2700"] [2024/05/06 19:57:29.295 +08:00] [INFO] [main.go:220] ["1 insert success: 2700"] reset and all metadata truncated in PD! PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371066c80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:24020, start at 2024-05-06 19:57:28.376689287 +0800 CST m=+5.436937540 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:28.384 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:28.370 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:28.370 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:57:29.384 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:120,start_key:7480000000000000ff785f720000000000fa,end_key:7480000000000000ff785f730000000000fa}] [checkpointTs=449576080080896003] [2024/05/06 19:57:29.384 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:120,start_key:7480000000000000ff785f720000000000fa,end_key:7480000000000000ff785f730000000000fa}] [checkpointTs=449576080080896003] [2024/05/06 19:57:29.484 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=5008] [appliedDDLCount=0] [currentCheckpoint=449576080080896003] [2024/05/06 19:57:29.484 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 19:57:29.484 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 19:57:29.484 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 19:57:29.484 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/06 19:57:29.484 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 19:57:29.484 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 19:57:29.484 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 19:57:29.484 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 19:57:29.484 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 19:57:29.484 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/06 19:57:29.484 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/06 19:57:29.484 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/06 19:57:29.484 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 19:57:29.484 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 19:57:29.484 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/06 19:57:29.484 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] Apply redo log successfully [2024/05/06 19:57:29.351 +08:00] [INFO] [main.go:220] ["0 insert success: 2800"] [2024/05/06 19:57:29.353 +08:00] [INFO] [main.go:234] ["0 delete success: 1400"] [2024/05/06 19:57:29.467 +08:00] [INFO] [main.go:234] ["1 delete success: 1400"] [2024/05/06 19:57:29.468 +08:00] [INFO] [main.go:220] ["1 insert success: 2800"] [2024/05/06 19:57:29.526 +08:00] [INFO] [main.go:220] ["0 insert success: 2900"] check diff failed 8-th time, retry later check diff successfully [Mon May 6 19:57:30 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> count(*) 5000 + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371060740013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-59v7t, pid:10519, start at 2024-05-06 19:57:27.990466282 +0800 CST m=+5.286822457 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:27.997 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:27.965 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:27.965 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371060740013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-59v7t, pid:10519, start at 2024-05-06 19:57:27.990466282 +0800 CST m=+5.286822457 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:27.997 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:27.965 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:27.965 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371060480015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-59v7t, pid:10595, start at 2024-05-06 19:57:28.004283406 +0800 CST m=+5.236788064 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:28.011 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:28.004 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:28.004 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_remove_table_error/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/owner_remove_table_error/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371066c80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:24020, start at 2024-05-06 19:57:28.376689287 +0800 CST m=+5.436937540 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:28.384 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:28.370 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:28.370 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371069040014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:24091, start at 2024-05-06 19:57:28.533705781 +0800 CST m=+5.537810305 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:28.542 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:28.513 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:28.513 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 9-th time, retry later check_ts_forward 1dc3442d-acde-443c-a8dd-b67ebf61b0bf + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.25381.out cli tso query --pd=http://127.0.0.1:2379 [Mon May 6 19:57:33 CST 2024] <<<<<< START cdc server in owner_remove_table_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/OwnerRemoveTableError=1*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_remove_table_error.1195811960.out server --log-file /tmp/tidb_cdc_test/owner_remove_table_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_remove_table_error/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... changefeed is working normally rts: 449576081919574000->449576082443599900 checkpoint: 449576081919574000->449576082443599900 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2024/05/06 19:57:27.559 +08:00] [INFO] [mysql_ddl_sink.go:125] ["Execute DDL failed, but error can be ignored"] [startTs=449576080320233484] [ddl="CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT)"] [namespace=default] [changefeed=1dc3442d-acde-443c-a8dd-b67ebf61b0bf] [error="Error 1050 (42S01): Table 'ddl_reentrant.t4' already exists"] run task successfully ------------------------------------------ test ddl alter table ddl_reentrant.t3 exchange partition p0 with table ddl_reentrant.t4, is_reentrant: true restored_sql: ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4` check_ts_forward 1dc3442d-acde-443c-a8dd-b67ebf61b0bf + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40978.out cli unsafe resolve-lock --region=64 check diff failed 10-th time, retry later PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:57:34.815 +08:00] [INFO] [main.go:86] ["running ddl test: 3 addDropColumnDDL2"] [2024/05/06 19:57:35.015 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 19:57:35.022 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] + set +x + tso='449576082766561281 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576082766561281 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x try a VALID cdc server command [Mon May 6 19:57:35 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.2542225424.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:57:35.187 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 19:57:35.188 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 19:57:35.196 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 19:57:35.197 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 19:57:35.361 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/06 19:57:35.369 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 19:57:35.532 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 19:57:35.533 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 19:57:35.539 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 19:57:35.540 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] changefeed is working normally rts: 449576082718851100->449576082980995100 checkpoint: 449576082705743900->449576082980995100 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/06 19:57:33.899 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4`"] [duration=145.83446ms] [namespace=default] [changefeed=1dc3442d-acde-443c-a8dd-b67ebf61b0bf] run task successfully nonempty select id, val from test.availability1 where id=3 and val=3 run task successfully wait process cdc.test exit for 1-th time... [2024/05/06 19:57:35.702 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 19:57:35.712 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.41012.out cli unsafe resolve-lock --region=64 --ts=449576082040684548 [2024/05/06 19:57:35.879 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 19:57:35.880 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 19:57:35.880 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/06 19:57:35.881 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 19:57:36.050 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 19:57:36.057 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] check diff failed 11-th time, retry later PASS [2024/05/06 19:57:36.214 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 19:57:36.215 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 19:57:36.228 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 19:57:36.229 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] wait process cdc.test exit for 2-th time... coverage: 2.2% of statements in github.com/pingcap/tiflow/... resume changefeed 1dc3442d-acde-443c-a8dd-b67ebf61b0bf from 449576082705743886 check_ts_forward 1dc3442d-acde-443c-a8dd-b67ebf61b0bf [2024/05/06 19:57:36.378 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 19:57:36.402 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/06 19:57:36.549 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/06 19:57:36.550 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 19:57:36.574 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/06 19:57:36.575 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:57:36 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/823b6865-89f7-40c6-a433-79b3120cf022 {"id":"823b6865-89f7-40c6-a433-79b3120cf022","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996653} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc3f58431 823b6865-89f7-40c6-a433-79b3120cf022 /tidb/cdc/default/default/upstream/7365854480549626160 {"id":7365854480549626160,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/823b6865-89f7-40c6-a433-79b3120cf022 {"id":"823b6865-89f7-40c6-a433-79b3120cf022","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996653} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc3f58431 823b6865-89f7-40c6-a433-79b3120cf022 /tidb/cdc/default/default/upstream/7365854480549626160 {"id":7365854480549626160,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/823b6865-89f7-40c6-a433-79b3120cf022 {"id":"823b6865-89f7-40c6-a433-79b3120cf022","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996653} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc3f58431 823b6865-89f7-40c6-a433-79b3120cf022 /tidb/cdc/default/default/upstream/7365854480549626160 {"id":7365854480549626160,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... [2024/05/06 19:57:36.711 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/06 19:57:36.748 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/06 19:57:36.902 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/06 19:57:36.903 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 19:57:36.940 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/06 19:57:36.941 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 19:57:37.065 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit run test case test_expire_capture [Mon May 6 19:57:37 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.62756277.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:57:37.106 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/06 19:57:37.233 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/06 19:57:37.233 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 19:57:37.279 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/06 19:57:37.279 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 19:57:37.407 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/06 19:57:37.453 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/06 19:57:37.580 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/06 19:57:37.581 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] check diff failed 1-th time, retry later + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 1643 46 --:--:-- --:--:-- --:--:-- 1656 { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" [2024/05/06 19:57:37.624 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/06 19:57:37.625 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/06 19:57:37.755 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/06 19:57:37.800 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] check diff failed 12-th time, retry later [2024/05/06 19:57:37.918 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/06 19:57:37.919 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/06 19:57:37.965 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/06 19:57:37.965 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/06 19:57:38.082 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/06 19:57:38.127 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/06 19:57:38.244 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/06 19:57:38.245 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/06 19:57:38.294 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/06 19:57:38.295 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:57:38 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8667356-2674-4b0d-b3e7-851de53327c7 {"id":"f8667356-2674-4b0d-b3e7-851de53327c7","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996655} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc3ec8e49 f8667356-2674-4b0d-b3e7-851de53327c7 /tidb/cdc/default/default/upstream/7365854471319033937 {"id":7365854471319033937,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8667356-2674-4b0d-b3e7-851de53327c7 {"id":"f8667356-2674-4b0d-b3e7-851de53327c7","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996655} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc3ec8e49 f8667356-2674-4b0d-b3e7-851de53327c7 /tidb/cdc/default/default/upstream/7365854471319033937 {"id":7365854471319033937,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8667356-2674-4b0d-b3e7-851de53327c7 {"id":"f8667356-2674-4b0d-b3e7-851de53327c7","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996655} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc3ec8e49 f8667356-2674-4b0d-b3e7-851de53327c7 /tidb/cdc/default/default/upstream/7365854471319033937 {"id":7365854471319033937,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/06 19:57:38.408 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/06 19:57:38.456 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/06 19:57:38.569 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/06 19:57:38.570 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/06 19:57:38.623 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/06 19:57:38.624 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/06 19:57:38.738 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/06 19:57:38.791 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/06 19:57:38.916 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/05/06 19:57:38.917 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/06 19:57:38.963 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/06 19:57:38.964 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/06 19:57:39.076 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/05/06 19:57:39.125 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] table owner_remove_table_error.finished_mark not exists for 1-th check, retry later [2024/05/06 19:57:39.233 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] [2024/05/06 19:57:39.234 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/05/06 19:57:39.285 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/05/06 19:57:39.286 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] check diff successfully [2024/05/06 19:57:39.388 +08:00] [INFO] [main.go:220] ["1 insert success: 2700"] [2024/05/06 19:57:39.454 +08:00] [INFO] [main.go:220] ["0 insert success: 2700"] [2024/05/06 19:57:39.545 +08:00] [INFO] [main.go:234] ["1 delete success: 1400"] [2024/05/06 19:57:39.546 +08:00] [INFO] [main.go:220] ["1 insert success: 2800"] [2024/05/06 19:57:39.621 +08:00] [INFO] [main.go:234] ["0 delete success: 1400"] [2024/05/06 19:57:39.622 +08:00] [INFO] [main.go:220] ["0 insert success: 2800"] [2024/05/06 19:57:39.718 +08:00] [INFO] [main.go:220] ["1 insert success: 2900"] [2024/05/06 19:57:39.797 +08:00] [INFO] [main.go:220] ["0 insert success: 2900"] wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check_ts_forward 1dc3442d-acde-443c-a8dd-b67ebf61b0bf wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:57:40 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0009b9950 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576082960023556 MinTableBarrierTs:449576084270743558 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576082960023556, checkpointTs: 449576082960023556, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576082960023556, checkpointTs: 449576082960023556, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576082960023556, checkpointTs: 449576082960023556, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/066f73c2-463e-4641-9439-17ffbb93fcdf {"id":"066f73c2-463e-4641-9439-17ffbb93fcdf","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996657} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077edce 066f73c2-463e-4641-9439-17ffbb93fcdf /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576082960023556,"min-table-barrier-ts":449576084270743558,"admin-job-type":0} /tidb/cdc/default/default/task/position/066f73c2-463e-4641-9439-17ffbb93fcdf/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0009b9950 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576082960023556 MinTableBarrierTs:449576084270743558 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576082960023556, checkpointTs: 449576082960023556, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576082960023556, checkpointTs: 449576082960023556, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576082960023556, checkpointTs: 449576082960023556, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/066f73c2-463e-4641-9439-17ffbb93fcdf {"id":"066f73c2-463e-4641-9439-17ffbb93fcdf","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996657} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077edce 066f73c2-463e-4641-9439-17ffbb93fcdf /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576082960023556,"min-table-barrier-ts":449576084270743558,"admin-job-type":0} /tidb/cdc/default/default/task/position/066f73c2-463e-4641-9439-17ffbb93fcdf/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0009b9950 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576082960023556 MinTableBarrierTs:449576084270743558 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576082960023556, checkpointTs: 449576082960023556, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576082960023556, checkpointTs: 449576082960023556, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576082960023556, checkpointTs: 449576082960023556, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/066f73c2-463e-4641-9439-17ffbb93fcdf {"id":"066f73c2-463e-4641-9439-17ffbb93fcdf","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996657} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077edce 066f73c2-463e-4641-9439-17ffbb93fcdf /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576082960023556,"min-table-barrier-ts":449576084270743558,"admin-job-type":0} /tidb/cdc/default/default/task/position/066f73c2-463e-4641-9439-17ffbb93fcdf/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' check diff failed 13-th time, retry later wait process cdc.test exit for 3-th time... table owner_remove_table_error.finished_mark not exists for 2-th check, retry later } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 1384 0 --:--:-- --:--:-- --:--:-- 1389 { "version": "v8.1.0-alpha-23-ge1a27adc6", "git_hash": "e1a27adc69acc9d50db0c3fea2945bbd6d983e30", "id": "2c0f81e3-ba11-46d3-9cde-cc66a4c96b79", "pid": 40112, "is_owner": true }wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:57:41 CST 2024] <<<<<< run test case split_region success! >>>>>> "is-owner": true, run task successfully Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... check diff failed 14-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value Starting Upstream PD... wait process cdc.test exit for 3-th time... changefeed is working normally rts: 449576084291715100->449576084816265200 checkpoint: 449576084291715100->449576084816265200 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/06 19:57:36.703 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4`"] [duration=143.462301ms] [namespace=default] [changefeed=1dc3442d-acde-443c-a8dd-b67ebf61b0bf] run task successfully Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... table owner_remove_table_error.finished_mark exists wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:57:42 CST 2024] <<<<<< run test case cli_tls_with_auth success! >>>>>> check diff successfully wait process cdc.test exit for 2-th time... owner pid: 6280 owner id 066f73c2-463e-4641-9439-17ffbb93fcdf process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v '066f73c2-463e-4641-9439-17ffbb93fcdf' /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077edce run task successfully process status: S nonempty select id, val from test.availability1 where id=2 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 3-th time... check diff failed 15-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:57:44 CST 2024] <<<<<< run test case ddl_reentrant success! >>>>>> wait process cdc.test exit for 3-th time... [2024/05/06 19:57:45.030 +08:00] [INFO] [main.go:86] ["running ddl test: 4 modifyColumnDDL"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:57:45 CST 2024] <<<<<< run test case owner_remove_table_error success! >>>>>> [2024/05/06 19:57:45.219 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 19:57:45.222 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 19:57:45.389 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 19:57:45.390 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 19:57:45.391 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 19:57:45.393 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 19:57:45.559 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 19:57:45.562 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] nonempty select id, val from test.availability1 where id=2 and val=22 run task successfully [2024/05/06 19:57:45.732 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 19:57:45.734 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 19:57:45.734 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 19:57:45.735 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] wait process cdc.test exit for 1-th time... Verifying downstream PD is started... [2024/05/06 19:57:45.917 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/06 19:57:45.923 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 19:57:46.073 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 19:57:46.075 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/06 19:57:46.083 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 19:57:46.084 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 19:57:46.236 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 19:57:46.247 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 19:57:46.393 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 19:57:46.394 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] wait process cdc.test exit for 2-th time... [2024/05/06 19:57:46.412 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 19:57:46.413 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 19:57:46.559 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 19:57:46.579 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] check diff failed 16-th time, retry later [2024/05/06 19:57:46.727 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 19:57:46.729 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/06 19:57:46.741 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/06 19:57:46.742 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 19:57:46.895 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/06 19:57:46.897 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 19:57:47.050 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/06 19:57:47.051 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 19:57:47.051 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 19:57:47.053 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/06 19:57:47.211 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/06 19:57:47.219 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/06 19:57:47.378 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/06 19:57:47.379 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 19:57:47.386 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 19:57:47.388 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit sql_check run sql_check, 127.0.0.1 run task successfully run test case test_stop_processor [Mon May 6 19:57:47 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.64916493.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_stop_processor.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_stop_processor --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:57:47.539 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/06 19:57:47.552 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/06 19:57:47.697 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/06 19:57:47.698 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/06 19:57:47.708 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/06 19:57:47.709 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/06 19:57:47.855 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/06 19:57:47.861 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/06 19:57:48.007 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/06 19:57:48.008 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/06 19:57:48.021 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/06 19:57:48.023 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/06 19:57:48.203 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/06 19:57:48.208 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/06 19:57:48.371 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/06 19:57:48.372 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/06 19:57:48.377 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/06 19:57:48.379 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/06 19:57:48.533 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/06 19:57:48.555 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371188900018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-54cks, pid:23458, start at 2024-05-06 19:57:46.964778881 +0800 CST m=+5.309448125 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:46.971 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:46.967 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:46.967 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371188900018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-54cks, pid:23458, start at 2024-05-06 19:57:46.964778881 +0800 CST m=+5.309448125 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:46.971 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:46.967 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:46.967 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37118b8c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-54cks, pid:23534, start at 2024-05-06 19:57:47.142071449 +0800 CST m=+5.423261567 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:47.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:47.107 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:47.107 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 19:57:48.706 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/06 19:57:48.707 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/06 19:57:48.744 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/06 19:57:48.745 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/06 19:57:48.885 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/06 19:57:48.918 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... [2024/05/06 19:57:49.049 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/06 19:57:49.051 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/06 19:57:49.097 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/06 19:57:49.099 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/05/06 19:57:49.210 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/05/06 19:57:49.270 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/05/06 19:57:49.368 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/05/06 19:57:49.369 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/05/06 19:57:49.433 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/05/06 19:57:49.435 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] wait process cdc.test exit for 2-th time... [2024/05/06 19:57:49.522 +08:00] [INFO] [main.go:220] ["0 insert success: 2700"] [2024/05/06 19:57:49.599 +08:00] [INFO] [main.go:220] ["1 insert success: 2700"] [2024/05/06 19:57:49.672 +08:00] [INFO] [main.go:234] ["0 delete success: 1400"] [2024/05/06 19:57:49.673 +08:00] [INFO] [main.go:220] ["0 insert success: 2800"] [2024/05/06 19:57:49.777 +08:00] [INFO] [main.go:220] ["1 insert success: 2800"] [2024/05/06 19:57:49.778 +08:00] [INFO] [main.go:234] ["1 delete success: 1400"] [2024/05/06 19:57:49.869 +08:00] [INFO] [main.go:220] ["0 insert success: 2900"] wait process cdc.test exit for 3-th time... [2024/05/06 19:57:49.991 +08:00] [INFO] [main.go:220] ["1 insert success: 2900"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:57:50 CST 2024] <<<<<< run test case sink_hang success! >>>>>> + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.24803.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:57:50 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b11170 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576085581725702 MinTableBarrierTs:449576087167434753 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576085581725702, checkpointTs: 449576085581725702, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576085581725702, checkpointTs: 449576085581725702, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576085581725702, checkpointTs: 449576085581725702, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a05a8d1e-21a2-43d7-bbfc-fa606d6009f5 {"id":"a05a8d1e-21a2-43d7-bbfc-fa606d6009f5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996667} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077ee47 a05a8d1e-21a2-43d7-bbfc-fa606d6009f5 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576085581725702,"min-table-barrier-ts":449576087167434753,"admin-job-type":0} /tidb/cdc/default/default/task/position/a05a8d1e-21a2-43d7-bbfc-fa606d6009f5/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b11170 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576085581725702 MinTableBarrierTs:449576087167434753 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576085581725702, checkpointTs: 449576085581725702, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576085581725702, checkpointTs: 449576085581725702, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576085581725702, checkpointTs: 449576085581725702, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a05a8d1e-21a2-43d7-bbfc-fa606d6009f5 {"id":"a05a8d1e-21a2-43d7-bbfc-fa606d6009f5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996667} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077ee47 a05a8d1e-21a2-43d7-bbfc-fa606d6009f5 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576085581725702,"min-table-barrier-ts":4495+ grep -q 'failed to get info:' 76087167434753,"admin-job-type":0} /tidb/cdc/default/default/task/position/a05a8d1e-21a2-43d7-bbfc-fa606d6009f5/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15d744e6-9542-4a9e-820f-73b6ebaa1984 {UpstreamID:7365853495484688586 Namespace:default ID:15d744e6-9542-4a9e-820f-73b6ebaa1984 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:53:54.37838987 +0800 CST StartTs:449576023807492098 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b11170 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576025288605699} {CheckpointTs:449576085581725702 MinTableBarrierTs:449576087167434753 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449576085581725702, checkpointTs: 449576085581725702, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576085581725702, checkpointTs: 449576085581725702, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576085581725702, checkpointTs: 449576085581725702, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a05a8d1e-21a2-43d7-bbfc-fa606d6009f5 {"id":"a05a8d1e-21a2-43d7-bbfc-fa606d6009f5","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996667} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc077ee47 a05a8d1e-21a2-43d7-bbfc-fa606d6009f5 /tidb/cdc/default/default/changefeed/info/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"upstream-id":7365853495484688586,"namespace":"default","changefeed-id":"15d744e6-9542-4a9e-820f-73b6ebaa1984","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:53:54.37838987+08:00","start-ts":449576023807492098,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576025288605699} /tidb/cdc/default/default/changefeed/status/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":449576085581725702,"min-table-barrier-ts":4495+ grep -q 'etcd info' 76087167434753,"admin-job-type":0} /tidb/cdc/default/default/task/position/a05a8d1e-21a2-43d7-bbfc-fa606d6009f5/15d744e6-9542-4a9e-820f-73b6ebaa1984 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365853495484688586 {"id":7365853495484688586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } "is-owner": true, run task successfully [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + set +x + tso='449576087300866054 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576087300866054 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:57:52 CST 2024] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.2484624848.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) owner pid: 6496 owner id a05a8d1e-21a2-43d7-bbfc-fa606d6009f5 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_cdc_cluster/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3711f6040005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:12849, start at 2024-05-06 19:57:53.926780254 +0800 CST m=+5.130353230 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:53.933 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:53.921 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:53.921 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3711f6040005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:12849, start at 2024-05-06 19:57:53.926780254 +0800 CST m=+5.130353230 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:53.933 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:53.921 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:53.921 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3711f6b40005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:12930, start at 2024-05-06 19:57:53.969949676 +0800 CST m=+5.097203734 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:59:53.976 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:57:53.965 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:47:53.965 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // cache [Pipeline] } [Pipeline] // dir [2024/05/06 19:57:55.184 +08:00] [INFO] [main.go:86] ["running ddl test: 5 addDropIndexDDL"] [2024/05/06 19:57:55.357 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 19:57:55.359 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [Pipeline] } changefeed id: 15d744e6-9542-4a9e-820f-73b6ebaa1984 curl -s -d "cf-id=15d744e6-9542-4a9e-820f-73b6ebaa1984&admin-job=1" http://127.0.0.1:8300/capture/owner/admin | grep true "status": true, run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 89 100 35 100 54 452 697 --:--:-- --:--:-- --:--:-- 701 { "status": true, "message": "" }nonempty select id, val from test.availability1 where id=4 and val=4 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [2024/05/06 19:57:55.502 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 19:57:55.503 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 19:57:55.504 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 19:57:55.505 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 19:57:55.648 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 19:57:55.648 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:57:55 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a8d8c9a-4f89-4135-bad4-0fea18e91149 {"id":"4a8d8c9a-4f89-4135-bad4-0fea18e91149","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996672} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc439b136 4a8d8c9a-4f89-4135-bad4-0fea18e91149 /tidb/cdc/default/default/upstream/7365854559280228229 {"id":7365854559280228229,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a8d8c9a-4f89-4135-bad4-0fea18e91149 {"id":"4a8d8c9a-4f89-4135-bad4-0fea18e91149","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996672} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc439b136 4a8d8c9a-4f89-4135-bad4-0fea18e91149 /tidb/cdc/default/default/upstream/7365854559280228229 {"id":7365854559280228229,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a8d8c9a-4f89-4135-bad4-0fea18e91149 {"id":"4a8d8c9a-4f89-4135-bad4-0fea18e91149","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996672} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc439b136 4a8d8c9a-4f89-4135-bad4-0fea18e91149 /tidb/cdc/default/default/upstream/7365854559280228229 {"id":7365854559280228229,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.24901.out cli changefeed create --start-ts=449576087300866054 --sink-uri=mysql://root@127.0.0.1:3306/ [Pipeline] // container [Pipeline] } [2024/05/06 19:57:55.825 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 19:57:55.827 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 19:57:55.830 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 19:57:55.831 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [Pipeline] // withEnv [Pipeline] } [Pipeline] // node Create changefeed successfully! ID: 7348d919-5e4f-4871-9082-94ac98f7ba3b Info: {"upstream_id":7365854559280228229,"namespace":"default","id":"7348d919-5e4f-4871-9082-94ac98f7ba3b","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-06T19:57:55.988019158+08:00","start_ts":449576087300866054,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576087300866054,"checkpoint_ts":449576087300866054,"checkpoint_time":"2024-05-06 19:57:50.917"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... [Pipeline] } [Pipeline] // podTemplate [2024/05/06 19:57:56.034 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/06 19:57:56.048 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [2024/05/06 19:57:56.254 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 19:57:56.256 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/06 19:57:56.268 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 19:57:56.271 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 19:57:56.474 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 19:57:56.495 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 19:57:56.705 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 19:57:56.708 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] start tidb cluster in /tmp/tidb_cdc_test/multi_cdc_cluster Starting Upstream PD... [2024/05/06 19:57:56.727 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 19:57:56.730 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 19:57:56.944 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [Mon May 6 19:57:57 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file_large_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file_large_value.1429814300.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... [2024/05/06 19:57:56.993 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/06 19:57:57.221 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 19:57:57.225 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] nonempty select id, val from test.availability1 where id=4 and val=4 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later [2024/05/06 19:57:57.308 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 19:57:57.312 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/06 19:57:57.502 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/06 19:57:57.663 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_basic/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:57:55 CST 2024] <<<<<< run test case kafka_simple_basic success! >>>>>> [2024/05/06 19:57:57.796 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 19:57:57.801 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/06 19:57:57.983 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 19:57:57.986 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/06 19:57:58.000 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/06 19:57:58.225 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 19:57:58.228 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/06 19:57:58.277 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] + set +x [2024/05/06 19:57:58.519 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/06 19:57:58.615 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 19:57:58.621 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:57:58 CST 2024] <<<<<< run test case kafka_simple_basic_avro success! >>>>>> [2024/05/06 19:57:58.891 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/06 19:57:58.898 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/06 19:57:59.043 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/06 19:57:59.233 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/06 19:57:59.433 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/06 19:57:59.437 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/06 19:57:59.583 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/06 19:57:59.590 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/06 19:57:59.883 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/06 19:57:59.937 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] valid ~~~ running cdc Failed to start cdc, the usage tips should be printed 1st test case cdc_server_tips success! try an INVALID cdc server command [Mon May 6 19:57:58 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ true != \n\o ]] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.2552625528.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default --pd None + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:58:00 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef6eeda6-4c22-4062-bbdb-cb53ae9bf713 {"id":"ef6eeda6-4c22-4062-bbdb-cb53ae9bf713","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996677} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc4518b2e ef6eeda6-4c22-4062-bbdb-cb53ae9bf713 /tidb/cdc/default/default/upstream/7365854585040312406 {"id":7365854585040312406,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef6eeda6-4c22-4062-bbdb-cb53ae9bf713 {"id":"ef6eeda6-4c22-4062-bbdb-cb53ae9bf713","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996677} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc4518b2e ef6eeda6-4c22-4062-bbdb-cb53ae9bf713 /tidb/cdc/default/default/upstream/7365854585040312406 {"id":7365854585040312406,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef6eeda6-4c22-4062-bbdb-cb53ae9bf713 {"id":"ef6eeda6-4c22-4062-bbdb-cb53ae9bf713","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996677} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc4518b2e ef6eeda6-4c22-4062-bbdb-cb53ae9bf713 /tidb/cdc/default/default/upstream/7365854585040312406 {"id":7365854585040312406,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } nonempty select id, val from test.availability1 where id=4 and val=4 run task successfully test_stop_processor pass [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ***************** properties ***************** "readproportion"="0" "workload"="core" "fieldlength"="15000" "dotransactions"="false" "mysql.host"="127.0.0.1" "mysql.port"="4000" "mysql.user"="root" "requestdistribution"="uniform" "mysql.db"="consistent_replicate_storage_file_large_value" "threadcount"="10" "fieldlengthdistribution"="constant" "updateproportion"="0" "scanproportion"="0" "readallfields"="true" "insertproportion"="0" "fieldcount"="1000" "recordcount"="100" "operationcount"="0" ********************************************** wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:58:02 CST 2024] <<<<<< run test case kafka_simple_handle_key_only success! >>>>>> Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:58:03 CST 2024] <<<<<< run test case availability success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:58:05.596 +08:00] [INFO] [main.go:78] ["runDDLTest take 1m1.570999345s"] table mark.finish_mark_0 exists table mark.finish_mark_1 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_1 not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3712d5800015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-b2lwz, pid:11842, start at 2024-05-06 19:58:08.268707387 +0800 CST m=+5.207762803 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:00:08.276 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:58:08.274 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:48:08.274 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3712d5800015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-b2lwz, pid:11842, start at 2024-05-06 19:58:08.268707387 +0800 CST m=+5.207762803 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:00:08.276 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:58:08.274 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:48:08.274 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3712d5580014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-b2lwz, pid:11920, start at 2024-05-06 19:58:08.247312144 +0800 CST m=+5.137822291 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:00:08.254 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:58:08.214 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:48:08.214 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_cdc_cluster/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:58:09 CST 2024] <<<<<< run test case kafka_simple_handle_key_only_avro success! >>>>>> table mark.finish_mark_1 not exists for 3-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.13263.out cli tso query --pd=http://127.0.0.1:2379 Run finished, takes 9.222683086s INSERT_ERROR - Takes(s): 8.1, Count: 100, OPS: 12.3, Avg(us): 342535, Min(us): 195629, Max(us): 629066, 95th(us): 521000, 99th(us): 570000 table mark.finish_mark_1 not exists for 4-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_claim_check/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:58:12 CST 2024] <<<<<< run test case kafka_simple_claim_check success! >>>>>> table consistent_replicate_storage_file_large_value.usertable exists table consistent_replicate_storage_file_large_value.check1 not exists for 1-th check, retry later + set +x + tso='449576092742189058 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576092742189058 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies + source /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/http_proxies + CDC_BINARY=cdc.test + SINK_TYPE=mysql + TEST_HOST_LIST=(UP_TIDB_HOST UP_PD_HOST_{1..3} UP_TIKV_HOST_{1..3}) + lan_addrs=($(hostname -I)) ++ hostname -I + lan_addr=10.233.66.114 + export UP_TIDB_HOST=10.233.66.114 UP_PD_HOST_1=10.233.66.114 UP_PD_HOST_2=10.233.66.114 UP_PD_HOST_3=10.233.66.114 UP_TIKV_HOST_1=10.233.66.114 UP_TIKV_HOST_2=10.233.66.114 UP_TIKV_HOST_3=10.233.66.114 + UP_TIDB_HOST=10.233.66.114 + UP_PD_HOST_1=10.233.66.114 + UP_PD_HOST_2=10.233.66.114 + UP_PD_HOST_3=10.233.66.114 + UP_TIKV_HOST_1=10.233.66.114 + UP_TIKV_HOST_2=10.233.66.114 + UP_TIKV_HOST_3=10.233.66.114 + proxy_pid= ++ shuf -i 10081-20081 -n1 + proxy_port=14727 + trap 'stop_tidb_cluster && stop_proxy' EXIT + '[' mysql == mysql ']' + prepare + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIDB_HOST + echo 'UP_TIDB_HOST 10.233.66.114' UP_TIDB_HOST 10.233.66.114 + case $(printenv $host) in ++ printenv UP_TIDB_HOST + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_1 + echo 'UP_PD_HOST_1 10.233.66.114' UP_PD_HOST_1 10.233.66.114 + case $(printenv $host) in ++ printenv UP_PD_HOST_1 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_2 + echo 'UP_PD_HOST_2 10.233.66.114' UP_PD_HOST_2 10.233.66.114 + case $(printenv $host) in ++ printenv UP_PD_HOST_2 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_3 + echo 'UP_PD_HOST_3 10.233.66.114' UP_PD_HOST_3 10.233.66.114 + case $(printenv $host) in ++ printenv UP_PD_HOST_3 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_1 + echo 'UP_TIKV_HOST_1 10.233.66.114' UP_TIKV_HOST_1 10.233.66.114 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_1 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_2 + echo 'UP_TIKV_HOST_2 10.233.66.114' UP_TIKV_HOST_2 10.233.66.114 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_2 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_3 + echo 'UP_TIKV_HOST_3 10.233.66.114' UP_TIKV_HOST_3 10.233.66.114 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_3 + rm -rf /tmp/tidb_cdc_test/http_proxies + mkdir -p /tmp/tidb_cdc_test/http_proxies + stop_tidb_cluster [Mon May 6 19:58:13 CST 2024] <<<<<< START cdc server in multi_cdc_cluster case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.1330713309.out server --log-file /tmp/tidb_cdc_test/multi_cdc_cluster/cdcmult_cdc.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_cdc_cluster/cdc_datamult_cdc.server1 --cluster-id test1 --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_1 exists table mark.finish_mark_2 not exists for 1-th check, retry later table consistent_replicate_storage_file_large_value.check1 exists table consistent_replicate_storage_file_large_value.t2 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:58:15 CST 2024] <<<<<< run test case kafka_simple_claim_check_avro success! >>>>>> table mark.finish_mark_2 not exists for 2-th check, retry later table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark not exists for 1-th check, retry later table ddl_attributes.finish_mark not exists for 2-th check, retry later + start_tidb_cluster --workdir /tmp/tidb_cdc_test/http_proxies The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:58:16 GMT < Content-Length: 819 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/27b6ba42-14b5-41c4-9694-3b6b28b5f06d {"id":"27b6ba42-14b5-41c4-9694-3b6b28b5f06d","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996693} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22318f4dc4896a48 27b6ba42-14b5-41c4-9694-3b6b28b5f06d /tidb/cdc/test1/default/upstream/7365854650878381927 {"id":7365854650878381927,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/27b6ba42-14b5-41c4-9694-3b6b28b5f06d {"id":"27b6ba42-14b5-41c4-9694-3b6b28b5f06d","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996693} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22318f4dc4896a48 27b6ba42-14b5-41c4-9694-3b6b28b5f06d /tidb/cdc/test1/default/upstream/7365854650878381927 {"id":7365854650878381927,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/27b6ba42-14b5-41c4-9694-3b6b28b5f06d {"id":"27b6ba42-14b5-41c4-9694-3b6b28b5f06d","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996693} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22318f4dc4896a48 27b6ba42-14b5-41c4-9694-3b6b28b5f06d /tidb/cdc/test1/default/upstream/7365854650878381927 {"id":7365854650878381927,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:58:16 CST 2024] <<<<<< START cdc server in multi_cdc_cluster case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.1336213364.out server --log-file /tmp/tidb_cdc_test/multi_cdc_cluster/cdcmult_cdc.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_cdc_cluster/cdc_datamult_cdc.server2 --cluster-id test2 --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_storage_file_large_value.t2 not exists for 2-th check, retry later invalid ~~~ running cdc Failed to start cdc, the usage tips should be printed 2nd test case cdc_server_tips success! [Mon May 6 19:58:18 CST 2024] <<<<<< run all test cases cdc_server_tips success! >>>>>> table mark.finish_mark_2 not exists for 3-th check, retry later table ddl_attributes.finish_mark not exists for 3-th check, retry later table consistent_replicate_storage_file_large_value.t2 not exists for 3-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/http_proxies Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:58:19 GMT < Content-Length: 819 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/9fcffa47-3cdc-4b53-80ce-78216c0fd46b {"id":"9fcffa47-3cdc-4b53-80ce-78216c0fd46b","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996696} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22318f4dc4896a72 9fcffa47-3cdc-4b53-80ce-78216c0fd46b /tidb/cdc/test2/default/upstream/7365854650878381927 {"id":7365854650878381927,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/9fcffa47-3cdc-4b53-80ce-78216c0fd46b {"id":"9fcffa47-3cdc-4b53-80ce-78216c0fd46b","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996696} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22318f4dc4896a72 9fcffa47-3cdc-4b53-80ce-78216c0fd46b /tidb/cdc/test2/default/upstream/7365854650878381927 {"id":7365854650878381927,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/9fcffa47-3cdc-4b53-80ce-78216c0fd46b {"id":"9fcffa47-3cdc-4b53-80ce-78216c0fd46b","address":"127.0.0.1:8301","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996696} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22318f4dc4896a72 9fcffa47-3cdc-4b53-80ce-78216c0fd46b /tidb/cdc/test2/default/upstream/7365854650878381927 {"id":7365854650878381927,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.13416.out cli changefeed create --start-ts=449576092742189058 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server http://127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_cdc_cluster/conf/changefeed1.toml Create changefeed successfully! ID: 58174b30-6219-474d-bf92-b0103871cac2 Info: {"upstream_id":7365854650878381927,"namespace":"default","id":"58174b30-6219-474d-bf92-b0103871cac2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:58:19.934877628+08:00","start_ts":449576092742189058,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["test.multi_cdc1"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576092742189058,"checkpoint_ts":449576092742189058,"checkpoint_time":"2024-05-06 19:58:11.674"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_2 not exists for 4-th check, retry later table ddl_attributes.finish_mark not exists for 4-th check, retry later table consistent_replicate_storage_file_large_value.t2 not exists for 4-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.13453.out cli changefeed create --start-ts=449576092742189058 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server http://127.0.0.1:8301 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_cdc_cluster/conf/changefeed2.toml Create changefeed successfully! ID: 8052e949-0b15-4fb5-8ef6-229496e2f062 Info: {"upstream_id":7365854650878381927,"namespace":"default","id":"8052e949-0b15-4fb5-8ef6-229496e2f062","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:58:21.817468002+08:00","start_ts":449576092742189058,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["test.multi_cdc2"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576092742189058,"checkpoint_ts":449576092742189058,"checkpoint_time":"2024-05-06 19:58:11.674"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_2 not exists for 5-th check, retry later table ddl_attributes.finish_mark not exists for 5-th check, retry later table consistent_replicate_storage_file_large_value.t2 not exists for 5-th check, retry later + set +x table test.multi_cdc1 exists table test.multi_cdc2 exists check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_adapter_compatibility/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:58:22 CST 2024] <<<<<< run test case canal_json_adapter_compatibility success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table mark.finish_mark_2 not exists for 6-th check, retry later table ddl_attributes.finish_mark not exists for 6-th check, retry later table consistent_replicate_storage_file_large_value.t2 not exists for 6-th check, retry later Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.66.114' (111) check diff successfully wait process cdc.test exit for 1-th time... table mark.finish_mark_2 not exists for 7-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_basic/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:58:26 CST 2024] <<<<<< run test case canal_json_basic success! >>>>>> wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... table consistent_replicate_storage_file_large_value.t2 exists check diff successfully wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:58:27 CST 2024] <<<<<< run test case multi_cdc_cluster success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.66.114' (111) wait process cdc.test exit for 2-th time... table mark.finish_mark_2 not exists for 8-th check, retry later table ddl_attributes.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 3-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:58:29 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file_large_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file_large_value.1461414616.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.66.114' (111) table mark.finish_mark_2 exists table mark.finish_mark_3 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/event_filter/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Mon May 6 19:58:30 CST 2024] <<<<<< run test case ddl_attributes success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_content_compatible/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:58:30 CST 2024] <<<<<< run test case canal_json_content_compatible success! >>>>>> table mark.finish_mark_3 not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371439bc0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-c3sn1, pid:7675, start at 2024-05-06 19:58:31.025784056 +0800 CST m=+5.271207133 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:00:31.032 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:58:31.023 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:48:31.023 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.66.114' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:58:32 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/eb372ed6-b55e-4e69-95d2-3a20d29e55a9 {UpstreamID:7365854585040312406 Namespace:default ID:eb372ed6-b55e-4e69-95d2-3a20d29e55a9 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:58:00.41554308 +0800 CST StartTs:449576089752961030 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003bd1830 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576089779437575} {CheckpointTs:449576092925165593 MinTableBarrierTs:449576098036973573 AdminJobType:noop} span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aeb85d06-8a23-4349-8554-36703489d3e0 {"id":"aeb85d06-8a23-4349-8554-36703489d3e0","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996709} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc4518c8c aeb85d06-8a23-4349-8554-36703489d3e0 /tidb/cdc/default/default/changefeed/info/eb372ed6-b55e-4e69-95d2-3a20d29e55a9 {"upstream-id":7365854585040312406,"namespace":"default","changefeed-id":"eb372ed6-b55e-4e69-95d2-3a20d29e55a9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:58:00.41554308+08:00","start-ts":449576089752961030,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576089779437575} /tidb/cdc/default/default/changefeed/status/eb372ed6-b55e-4e69-95d2-3a20d29e55a9 {"checkpoint-ts":449576092925165593,"min-table-barrier-ts":449576098036973573,"admin-job-type":0} /tidb/cdc/default/default/task/position/aeb85d06-8a23-4349-8554-36703489d3e0/eb372ed6-b55e-4e69-95d2-3a20d29e55a9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854585040312406 {"id":7365854585040312406,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/eb372ed6-b55e-4e69-95d2-3a20d29e55a9 {UpstreamID:7365854585040312406 Namespace:default ID:eb372ed6-b55e-4e69-95d2-3a20d29e55a9 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:58:00.41554308 +0800 CST StartTs:449576089752961030 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003bd1830 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576089779437575} {CheckpointTs:449576092925165593 MinTableBarrierTs:449576098036973573 AdminJobType:noop} span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aeb85d06-8a23-4349-8554-36703489d3e0 {"id":"aeb85d06-8a23-4349-8554-36703489d3e0","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996709} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc4518c8c aeb85d06-8a23-4349-8554-36703489d3e0 /tidb/cdc/default/default/changefeed/info/eb372ed6-b55e-4e69-95d2-3a20d29e55a9 {"upstream-id":7365854585040312406,"namespace":"default","changefeed-id":"eb372ed6-b55e-4e69-95d2-3a20d29e55a9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:58:00.41554308+08:00","start-ts":449576089752961030,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576089779437575} /tidb/cdc/default/default/changefeed/status/eb372ed6-b55e-4e69-95d2-3a20d29e55a9 {"checkpoint-ts":449576092925165593,"min-table-barrier-ts":449576098036973573,"admin-job-type":0} /tidb/cdc/default/default/task/position/aeb85d06-8a23-4349-8554-36703489d3e0/eb372ed6-b55e-4e69-95d2-3a20d29e55a9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854585040312406 {"id":7365854585040312406,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/eb372ed6-b55e-4e69-95d2-3a20d29e55a9 {UpstreamID:7365854585040312406 Namespace:default ID:eb372ed6-b55e-4e69-95d2-3a20d29e55a9 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:58:00.41554308 +0800 CST StartTs:449576089752961030 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003bd1830 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576089779437575} {CheckpointTs:449576092925165593 MinTableBarrierTs:449576098036973573 AdminJobType:noop} span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449576092925165593, checkpointTs: 449576092925165593, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aeb85d06-8a23-4349-8554-36703489d3e0 {"id":"aeb85d06-8a23-4349-8554-36703489d3e0","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996709} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc4518c8c aeb85d06-8a23-4349-8554-36703489d3e0 /tidb/cdc/default/default/changefeed/info/eb372ed6-b55e-4e69-95d2-3a20d29e55a9 {"upstream-id":7365854585040312406,"namespace":"default","changefeed-id":"eb372ed6-b55e-4e69-95d2-3a20d29e55a9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:58:00.41554308+08:00","start-ts":449576089752961030,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576089779437575} /tidb/cdc/default/default/changefeed/status/eb372ed6-b55e-4e69-95d2-3a20d29e55a9 {"checkpoint-ts":449576092925165593,"min-table-barrier-ts":449576098036973573,"admin-job-type":0} /tidb/cdc/default/default/task/position/aeb85d06-8a23-4349-8554-36703489d3e0/eb372ed6-b55e-4e69-95d2-3a20d29e55a9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854585040312406 {"id":7365854585040312406,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371439bc0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-c3sn1, pid:7675, start at 2024-05-06 19:58:31.025784056 +0800 CST m=+5.271207133 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:00:31.032 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:58:31.023 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:48:31.023 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37143bfc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-c3sn1, pid:7763, start at 2024-05-06 19:58:31.167593074 +0800 CST m=+5.365933542 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:00:31.174 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:58:31.167 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:48:31.167 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_proxies/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_proxies/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/http_proxies/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["10.233.66.114:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_proxies/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/http_proxies/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_topics/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:58:33 CST 2024] <<<<<< run test case multi_topics success! >>>>>> table mark.finish_mark_3 not exists for 3-th check, retry later + start_proxy + echo 'dumpling grpc packet to /tmp/tidb_cdc_test/http_proxies/test_proxy.log...' dumpling grpc packet to /tmp/tidb_cdc_test/http_proxies/test_proxy.log... + proxy_pid=9140 + echo 'proxy port: 14727' proxy port: 14727 + echo 'proxy pid: 9140' proxy pid: 9140 + sleep 5 + GO111MODULE=on + WORK_DIR=/tmp/tidb_cdc_test/http_proxies + go run /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/run-proxy.go --port=14727 go: downloading google.golang.org/grpc v1.62.1 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading golang.org/x/sys v0.19.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f go: downloading github.com/gorilla/websocket v1.5.1 go: downloading github.com/rs/cors v1.7.0 table mark.finish_mark_3 not exists for 4-th check, retry later go: downloading golang.org/x/text v0.14.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/avro_basic/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:58:37 CST 2024] <<<<<< run test case avro_basic success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container table mark.finish_mark_3 not exists for 5-th check, retry later [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 6-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:58:40 CST 2024] <<<<<< run test case canal_json_handle_key_only success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + export http_proxy=http://127.0.0.1:14727 + http_proxy=http://127.0.0.1:14727 + export https_proxy=http://127.0.0.1:14727 + https_proxy=http://127.0.0.1:14727 + echo 'try to connect pd cluster via proxy, pd addr: 10.233.66.114:2379' try to connect pd cluster via proxy, pd addr: 10.233.66.114:2379 + ensure 10 curl http://10.233.66.114:2379/ curl http://10.233.66.114:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:14727; Connection refused run task failed 1-th time, retry later table mark.finish_mark_3 not exists for 7-th check, retry later curl http://10.233.66.114:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:14727; Connection refused run task failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:58:43 CST 2024] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> table mark.finish_mark_3 not exists for 8-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table mark.finish_mark_3 exists table mark.finish_mark_4 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_claim_check/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:58:47 CST 2024] <<<<<< run test case canal_json_claim_check success! >>>>>> curl http://10.233.66.114:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:14727; Connection refused run task failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371517180002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:26564, start at 2024-05-06 19:58:45.191069797 +0800 CST m=+5.660569110 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:00:45.198 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:58:45.190 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:48:45.190 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371517180002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:26564, start at 2024-05-06 19:58:45.191069797 +0800 CST m=+5.660569110 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:00:45.198 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:58:45.190 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:48:45.190 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371518800014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:26638, start at 2024-05-06 19:58:45.326012649 +0800 CST m=+5.744835817 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:00:45.334 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:58:45.330 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:48:45.330 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/event_filter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/event_filter/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table mark.finish_mark_4 not exists for 2-th check, retry later time="2024-05-06T19:58:49+08:00" level=info msg="Listening on 127.0.0.1:14727" time="2024-05-06T19:58:49+08:00" level=info msg="Not intercepting TLS connections" [Mon May 6 19:58:49 CST 2024] <<<<<< START cdc server in event_filter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.2786727869.out server --log-file /tmp/tidb_cdc_test/event_filter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/event_filter/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_4 not exists for 3-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/open_protocol_claim_check/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:58:50 CST 2024] <<<<<< run test case open_protocol_claim_check success! >>>>>> table mark.finish_mark_4 not exists for 4-th check, retry later curl http://10.233.66.114:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 19 100 19 0 0 11452 0 --:--:-- --:--:-- --:--:-- 19000 404 page not found run task successfully + echo started proxy pid: 9140 started proxy pid: 9140 + echo started proxy at port: 14727 started proxy at port: 14727 + cd /tmp/tidb_cdc_test/http_proxies ++ run_cdc_cli_tso_query 10.233.66.114 2379 + pd_host=10.233.66.114 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://10.233.66.114:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.cli.12726.out cli tso query --pd=http://10.233.66.114:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:58:52 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bffe5282-50e5-4cc6-8323-b876bf01d26e {"id":"bffe5282-50e5-4cc6-8323-b876bf01d26e","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996730} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc517d933 bffe5282-50e5-4cc6-8323-b876bf01d26e /tidb/cdc/default/default/upstream/7365854808491071305 {"id":7365854808491071305,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bffe5282-50e5-4cc6-8323-b876bf01d26e {"id":"bffe5282-50e5-4cc6-8323-b876bf01d26e","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996730} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc517d933 bffe5282-50e5-4cc6-8323-b876bf01d26e /tidb/cdc/default/default/upstream/7365854808491071305 {"id":7365854808491071305,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bffe5282-50e5-4cc6-8323-b876bf01d26e {"id":"bffe5282-50e5-4cc6-8323-b876bf01d26e","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996730} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc517d933 bffe5282-50e5-4cc6-8323-b876bf01d26e /tidb/cdc/default/default/upstream/7365854808491071305 {"id":7365854808491071305,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.cli.27924.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/event_filter/conf/cf.toml Create changefeed successfully! ID: 3123bd67-f6a6-4851-a72c-018748fba0b8 Info: {"upstream_id":7365854808491071305,"namespace":"default","id":"3123bd67-f6a6-4851-a72c-018748fba0b8","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:58:53.411031498+08:00","start_ts":449576103651835910,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["event_filter.*"],"event_filters":[{"matcher":["event_filter.t1"],"ignore_event":["drop table","delete"],"ignore_sql":null,"ignore_insert_value_expr":"id = 2 or city = 'tokyo'","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_truncate"],"ignore_event":["truncate table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_alter"],"ignore_event":["alter table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""}]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576103651835910,"checkpoint_ts":449576103651835910,"checkpoint_time":"2024-05-06 19:58:53.291"} PASS coverage: 2.7% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_storage_basic/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:58:53 CST 2024] <<<<<< run test case canal_json_storage_basic success! >>>>>> ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2628 0 --:--:-- --:--:-- --:--:-- 2662 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 19:58:42.615","puller_resolved_ts":"2024-05-06 19:58:34.615","last_synced_ts":"2024-05-06 19:56:25.765","now_ts":"2024-05-06 19:58:43.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:58:42.615","puller_resolved_ts":"2024-05-06' '19:58:34.615","last_synced_ts":"2024-05-06' '19:56:25.765","now_ts":"2024-05-06' '19:58:43.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 16174 8.0 0.0 14326596 143456 ? Sl 19:56 0:12 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 16238 5.6 0.0 13997252 138752 ? Sl 19:56 0:08 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1714996728.730377,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0041a4a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714996728.7304802,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714996728.8686175,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0043b9880/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714996728.8686926,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714996729.6816318,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00221a1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714996729.6816819,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T19:58:53.524029+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011e2540/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:58:53.525129+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001062c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:58:53.610402+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e38700/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} check_redo_resolved_ts eb372ed6-b55e-4e69-95d2-3a20d29e55a9 449576103777927171 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/eb372ed6-b55e-4e69-95d2-3a20d29e55a9/meta global resolved ts 449576103017709580 not forward to 449576103777927171 run task failed 1-th time, retry later table mark.finish_mark_4 not exists for 5-th check, retry later + set +x + set +x failed proxying s2c rpc error: code = Unavailable desc = client disconnected + tso='449576103686438913 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576103686438913 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449576103686438913 + echo 'query start ts: 449576103686438913' query start ts: 449576103686438913 + run_cdc_server --workdir /tmp/tidb_cdc_test/http_proxies --binary cdc.test [Mon May 6 19:58:54 CST 2024] <<<<<< START cdc server in http_proxies case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.1276612768.out server --log-file /tmp/tidb_cdc_test/http_proxies/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_proxies/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to proxy 127.0.0.1 port 14727 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 14727 (#0) * Server auth using Basic with user 'ticdc' > GET http://127.0.0.1:8300/debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Proxy-Connection: Keep-Alive > < HTTP/1.1 502 Bad Gateway < Date: Mon, 06 May 2024 11:58:54 GMT < Content-Length: 0 < * Connection #0 to host 127.0.0.1 left intact + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled check_redo_resolved_ts eb372ed6-b55e-4e69-95d2-3a20d29e55a9 449576103777927171 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/eb372ed6-b55e-4e69-95d2-3a20d29e55a9/meta global resolved ts 449576103541997578 not forward to 449576103777927171 run task failed 2-th time, retry later table mark.finish_mark_4 not exists for 6-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/run.sh using Sink-Type: mysql... <<================= [Mon May 6 19:58:56 CST 2024] <<<<<< run test case canal_json_storage_partition_table success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to proxy 127.0.0.1 port 14727 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 14727 (#0) * Server auth using Basic with user 'ticdc' > GET http://127.0.0.1:8300/debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Proxy-Connection: Keep-Alive > < HTTP/1.1 200 OK < Content-Length: 831 < Content-Type: text/plain; charset=utf-8 < Date: Mon, 06 May 2024 11:58:57 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ca0d9619-0ed2-468a-b5aa-0e6b27828119 {"id":"ca0d9619-0ed2-468a-b5aa-0e6b27828119","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996735} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2fc18f4dc4e2138c ca0d9619-0ed2-468a-b5aa-0e6b27828119 /tidb/cdc/default/default/upstream/7365854749688082764 {"id":7365854749688082764,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.66.114:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ca0d9619-0ed2-468a-b5aa-0e6b27828119 {"id":"ca0d9619-0ed2-468a-b5aa-0e6b27828119","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996735} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2fc18f4dc4e2138c ca0d9619-0ed2-468a-b5aa-0e6b27828119 /tidb/cdc/default/default/upstream/7365854749688082764 {"id":7365854749688082764,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.66.114:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ca0d9619-0ed2-468a-b5aa-0e6b27828119 {"id":"ca0d9619-0ed2-468a-b5aa-0e6b27828119","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996735} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2fc18f4dc4e2138c ca0d9619-0ed2-468a-b5aa-0e6b27828119 /tidb/cdc/default/default/upstream/7365854749688082764 {"id":7365854749688082764,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.66.114:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + echo started cdc server successfully started cdc server successfully + SINK_URI=blackhole:/// + run_cdc_cli changefeed create --start-ts=449576103686438913 --sink-uri=blackhole:/// + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.cli.12822.out cli changefeed create --start-ts=449576103686438913 --sink-uri=blackhole:/// table event_filter.t1 does not exists table event_filter.t1 exists table event_filter.t_normal not exists for 1-th check, retry later table mark.finish_mark_4 not exists for 7-th check, retry later failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled Create changefeed successfully! ID: 9335ba05-33e0-4b7f-89a0-e91fa8211299 Info: {"upstream_id":7365854749688082764,"namespace":"default","id":"9335ba05-33e0-4b7f-89a0-e91fa8211299","sink_uri":"blackhole:///","create_time":"2024-05-06T19:58:58.481527635+08:00","start_ts":449576103686438913,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576103686438913,"checkpoint_ts":449576103686438913,"checkpoint_time":"2024-05-06 19:58:53.423"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... {"level":"warn","ts":"2024-05-06T19:58:59.524942+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011e2540/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:58:59.526447+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001062c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:58:59.611323+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e38700/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table event_filter.t_normal exists table event_filter.t_truncate not exists for 1-th check, retry later + set +x + sleep 5 check_redo_resolved_ts eb372ed6-b55e-4e69-95d2-3a20d29e55a9 449576103777927171 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/eb372ed6-b55e-4e69-95d2-3a20d29e55a9/meta run task successfully *************************** 1. row *************************** count(distinct region_id): 1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_tables_ddl/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` wait process cdc.test exit for 1-th time... Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide split_and_random_merge scale: 40 wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [2024/05/06 19:59:02.234 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 19:59:02.235 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2024/05/06 19:59:02.235 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=:6060] [2024/05/06 19:59:02.235 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 19:59:02.235 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449576098377498630] [resolvedTs=449576104603418625] [2024/05/06 19:59:02.236 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=909.43ยตs] [2024/05/06 19:59:02.236 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=853.45ยตs] [2024/05/06 19:59:02.236 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 19:59:02.236 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 19:59:02.236 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/06 19:59:02.242 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 19:59:02.246 +08:00] [INFO] [mysql.go:197] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 19:59:02.248 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:59:02.248 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:59:02.249 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:59:02.249 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:59:02.249 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:59:02.249 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:59:02.249 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:59:02.249 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:59:02.249 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:59:02.249 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:59:02.249 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:59:02.249 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:59:02.249 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:59:02.249 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:59:02.249 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:59:02.249 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 19:59:02.249 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 19:59:02.249 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 19:59:02.249 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 19:59:02.249 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 19:59:02.249 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 19:59:02.249 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 19:59:02.249 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 19:59:02.249 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 19:59:02.249 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 19:59:02.249 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 19:59:02.249 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 19:59:02.249 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 19:59:02.249 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 19:59:02.249 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 19:59:02.249 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 19:59:02.249 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 19:59:02.250 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 19:59:02.250 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 19:59:02.250 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/06 19:59:02.255 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 19:59:02.257 +08:00] [INFO] [mysql_ddl_sink.go:105] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/06 19:59:02.257 +08:00] [ERROR] [redo.go:268] ["ignore unsupported DDL"] [ddl="{\"StartTs\":449576098364653578,\"CommitTs\":449576098377498630,\"Query\":\"ALTER TABLE `consistent_replicate_storage_file_large_value`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_file_large_value`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_storage_file_large_value\",\"Table\":\"t1\",\"TableID\":110,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCE4735258E717 < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 11:59:02 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... table event_filter.t_truncate exists table event_filter.t_alter not exists for 1-th check, retry later [2024/05/06 19:59:02.458 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}] [checkpointTs=449576104603418625] [2024/05/06 19:59:02.458 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}] [checkpointTs=449576104603418625] table mark.finish_mark_4 not exists for 8-th check, retry later [2024/05/06 19:59:02.759 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}] [checkpointTs=449576104603418625] [2024/05/06 19:59:02.760 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}] [checkpointTs=449576104603418625] [2024/05/06 19:59:02.860 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=8] [appliedDDLCount=0] [currentCheckpoint=449576104603418625] [2024/05/06 19:59:02.860 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 19:59:02.860 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 19:59:02.860 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/06 19:59:02.860 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 19:59:02.860 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 19:59:02.860 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 19:59:02.860 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/06 19:59:02.860 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/06 19:59:02.860 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 19:59:02.860 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/06 19:59:02.860 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/06 19:59:02.860 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 19:59:02.860 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/06 19:59:02.860 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 19:59:02.860 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 19:59:02.860 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] Apply redo log successfully ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed check diff successfully [Mon May 6 19:59:03 CST 2024] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> count(*) 0 table mark.finish_mark_4 exists table mark.finish_mark not exists for 1-th check, retry later table event_filter.t_alter exists + check + services=($(cat $WORK_DIR/test_proxy.log | xargs -L1 dirname | sort | uniq)) ++ cat /tmp/tidb_cdc_test/http_proxies/test_proxy.log ++ xargs -L1 dirname ++ sort ++ uniq xargs: unmatched double quote; by default quotes are special to xargs unless you use the -0 option + service_type_count=5 + echo 'captured services: ' captured services: + echo . /grpc.health.v1.Health /pdpb.PD /resource_manager.ResourceManager '[2024/05' . /grpc.health.v1.Health /pdpb.PD /resource_manager.ResourceManager [2024/05 + '[' 5 -ge 2 ']' + check_logs /tmp/tidb_cdc_test/http_proxies ++ date + echo '[Mon May 6 19:59:04 CST 2024] <<<<<< run test case http_proxies success! >>>>>>' [Mon May 6 19:59:04 CST 2024] <<<<<< run test case http_proxies success! >>>>>> + stop_tidb_cluster failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Unavailable desc = client disconnected failed proxying s2c rpc error: code = Canceled desc = context canceled 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:59:05.526662+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011e2540/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:59:05.527526+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001062c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:59:05.612902+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e38700/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table event_filter.finish_mark exists check diff failed 1-th time, retry later table mark.finish_mark not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... *************************** 1. row *************************** count(distinct region_id): 4 check diff failed 2-th time, retry later table mark.finish_mark not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff successfully table mark.finish_mark not exists for 4-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:59:11.528358+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011e2540/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:59:11.529224+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001062c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} wait process cdc.test exit for 3-th time... {"level":"warn","ts":"2024-05-06T19:59:11.614216+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e38700/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:59:12 CST 2024] <<<<<< run test case event_filter success! >>>>>> table mark.finish_mark not exists for 5-th check, retry later Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) split_and_random_merge scale: 80 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:59:13.51533+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001062c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-06T19:59:13.515376+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T19:59:13.515368+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011e2540/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-06T19:59:13.515402+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide {"level":"warn","ts":"2024-05-06T19:59:13.60041+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e38700/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-06T19:59:13.600448+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} table mark.finish_mark not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCE4763DF6379B < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 11:59:14 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... table mark.finish_mark exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Mon May 6 19:59:17 CST 2024] <<<<<< run test case multi_source success! >>>>>> + stop_proxy + kill 9140 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/sequence/run.sh using Sink-Type: mysql... <<================= case sequence is skipped 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:59:17.52941+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011e2540/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:59:17.529725+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001062c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:59:17.614806+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e38700/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_s3 Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371723500006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:43489, start at 2024-05-06 19:59:18.748029105 +0800 CST m=+6.867636671 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:18.754 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:18.740 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:18.740 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371723500006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:43489, start at 2024-05-06 19:59:18.748029105 +0800 CST m=+6.867636671 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:18.754 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:18.740 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:18.740 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37170a440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-d65h9, pid:43564, start at 2024-05-06 19:59:17.1746587 +0800 CST m=+5.230880484 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:17.185 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:17.186 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:17.186 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/sql_mode/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/sql_mode/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/sql_mode ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/sql_mode + source /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/sql_mode/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/sql_mode + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + rm -rf /tmp/tidb_cdc_test/sql_mode + mkdir -p /tmp/tidb_cdc_test/sql_mode + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... [Mon May 6 19:59:22 CST 2024] <<<<<< START cdc server in multi_tables_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl.4490144903.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 *************************** 1. row *************************** count(distinct region_id): 3 Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:59:23.530503+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011e2540/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:59:23.53099+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001062c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:59:23.61588+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e38700/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":1714996763.7312455,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0041a4a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714996763.7312758,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714996763.8695362,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0043b9880/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714996763.869596,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0{"level":"warn","ts":1714996764.682692,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00221a1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714996764.6827242,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:59:25 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8fee47ea-c9ba-4ca9-918a-7b1bea03d509 {"id":"8fee47ea-c9ba-4ca9-918a-7b1bea03d509","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996762} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc5982e30 8fee47ea-c9ba-4ca9-918a-7b1bea03d509 /tidb/cdc/default/default/upstream/7365854946181955244 {"id":7365854946181955244,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8fee47ea-c9ba-4ca9-918a-7b1bea03d509 {"id":"8fee47ea-c9ba-4ca9-918a-7b1bea03d509","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996762} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc5982e30 8fee47ea-c9ba-4ca9-918a-7b1bea03d509 /tidb/cdc/default/default/upstream/7365854946181955244 {"id":7365854946181955244,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8fee47ea-c9ba-4ca9-918a-7b1bea03d509 {"id":"8fee47ea-c9ba-4ca9-918a-7b1bea03d509","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996762} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc5982e30 8fee47ea-c9ba-4ca9-918a-7b1bea03d509 /tidb/cdc/default/default/upstream/7365854946181955244 {"id":7365854946181955244,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: test-normal Info: {"upstream_id":7365854946181955244,"namespace":"default","id":"test-normal","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:59:25.587129761+08:00","start_ts":449576111240904705,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576111240904705,"checkpoint_ts":449576111240904705,"checkpoint_time":"2024-05-06 19:59:22.241"} Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7365854946181955244,"namespace":"default","id":"test-error-1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:59:25.786555982+08:00","start_ts":449576111240904705,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576111240904705,"checkpoint_ts":449576111240904705,"checkpoint_time":"2024-05-06 19:59:22.241"} Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7365854946181955244,"namespace":"default","id":"test-error-2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:59:25.986328823+08:00","start_ts":449576111240904705,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576111240904705,"checkpoint_ts":449576111240904705,"checkpoint_time":"2024-05-06 19:59:22.241"} ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 8978 0 --:--:-- --:--:-- --:--:-- 9037 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:57:04.450","puller_resolved_ts":"2024-05-06 19:57:04.450","last_synced_ts":"2024-05-06 19:56:57.650","now_ts":"2024-05-06 19:59:14.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:57:04.450","puller_resolved_ts":"2024-05-06' '19:57:04.450","last_synced_ts":"2024-05-06' '19:56:57.650","now_ts":"2024-05-06' '19:59:14.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ jq -r .info ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:57:04.450","puller_resolved_ts":"2024-05-06' '19:57:04.450","last_synced_ts":"2024-05-06' '19:56:57.650","now_ts":"2024-05-06' '19:59:14.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) [Pipeline] // cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // dir [Pipeline] } chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table region_merge.t1 exists check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_tables_ddl_test.t55 not exists for 1-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:59:29.531599+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011e2540/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:59:29.53171+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001062c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3717ba200003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:15956, start at 2024-05-06 19:59:28.395241618 +0800 CST m=+5.216092359 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:28.404 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:28.392 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:28.392 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release {"level":"warn","ts":"2024-05-06T19:59:29.617509+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e38700/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_tables_ddl_test.t55 not exists for 2-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3717ba200003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:15956, start at 2024-05-06 19:59:28.395241618 +0800 CST m=+5.216092359 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:28.404 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:28.392 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:28.392 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3717ba080010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:16040, start at 2024-05-06 19:59:28.405039735 +0800 CST m=+5.175791883 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:28.414 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:28.386 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:28.386 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:59:32 CST 2024] <<<<<< run test case region_merge success! >>>>>> 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' ++ jq -r .error_code + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_tables_ddl_test.t55 not exists for 3-th check, retry later wait process cdc.test exit for 1-th time... [Mon May 6 19:59:33 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_s3 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_s3.1740317405.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371814880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:29206, start at 2024-05-06 19:59:34.207803495 +0800 CST m=+5.194312681 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:34.214 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:34.178 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:34.178 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371814880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:29206, start at 2024-05-06 19:59:34.207803495 +0800 CST m=+5.194312681 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:34.214 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:34.178 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:34.178 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371815e80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:29271, start at 2024-05-06 19:59:34.309804646 +0800 CST m=+5.230471163 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:34.319 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:34.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:34.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sql_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sql_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table multi_tables_ddl_test.t55 not exists for 4-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:59:36 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5d15d489-e28c-4bff-a45c-324100d5dbd8 {"id":"5d15d489-e28c-4bff-a45c-324100d5dbd8","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996773} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc5c6472e 5d15d489-e28c-4bff-a45c-324100d5dbd8 /tidb/cdc/default/default/upstream/7365854998956661486 {"id":7365854998956661486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5d15d489-e28c-4bff-a45c-324100d5dbd8 {"id":"5d15d489-e28c-4bff-a45c-324100d5dbd8","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996773} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc5c6472e 5d15d489-e28c-4bff-a45c-324100d5dbd8 /tidb/cdc/default/default/upstream/7365854998956661486 {"id":7365854998956661486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5d15d489-e28c-4bff-a45c-324100d5dbd8 {"id":"5d15d489-e28c-4bff-a45c-324100d5dbd8","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996773} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc5c6472e 5d15d489-e28c-4bff-a45c-324100d5dbd8 /tidb/cdc/default/default/upstream/7365854998956661486 {"id":7365854998956661486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ***************** properties ***************** "mysql.user"="root" "updateproportion"="0" "readproportion"="0" "insertproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "fieldcount"="100" "mysql.db"="consistent_replicate_storage_s3" "workload"="core" "threadcount"="10" "dotransactions"="false" "readallfields"="true" "requestdistribution"="uniform" "operationcount"="0" "recordcount"="5000" "scanproportion"="0" ********************************************** table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later + trap stop_tidb_cluster EXIT + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 3306 + cd /tmp/tidb_cdc_test/sql_mode ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.30520.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371838fc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:15487, start at 2024-05-06 19:59:36.533418933 +0800 CST m=+5.183091054 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:36.539 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:36.511 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:36.511 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371838fc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:15487, start at 2024-05-06 19:59:36.533418933 +0800 CST m=+5.183091054 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:36.539 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:36.511 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:36.511 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d37183a680014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:15569, start at 2024-05-06 19:59:36.62123346 +0800 CST m=+5.221475652 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:36.627 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:36.602 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:36.602 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='449576115287621633 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576115287621633 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449576115287621633 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Mon May 6 19:59:39 CST 2024] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.3056230564.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later Run finished, takes 1.985273752s INSERT - Takes(s): 2.0, Count: 5000, OPS: 2529.6, Avg(us): 3684, Min(us): 1308, Max(us): 63836, 95th(us): 4000, 99th(us): 53000 table consistent_replicate_storage_s3.usertable exists table consistent_replicate_storage_s3.check1 not exists for 1-th check, retry later + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.16849.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449576115886096385 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576115886096385 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449576115886096385 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Mon May 6 19:59:41 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1689216894.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_storage_s3.check1 not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:59:42 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8f9d0fc-777a-4c62-bf0c-514ac07d9e98 {"id":"e8f9d0fc-777a-4c62-bf0c-514ac07d9e98","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996779} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc5e0fa38 e8f9d0fc-777a-4c62-bf0c-514ac07d9e98 /tidb/cdc/default/default/upstream/7365855022044863454 {"id":7365855022044863454,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8f9d0fc-777a-4c62-bf0c-514ac07d9e98 {"id":"e8f9d0fc-777a-4c62-bf0c-514ac07d9e98","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996779} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc5e0fa38 e8f9d0fc-777a-4c62-bf0c-514ac07d9e98 /tidb/cdc/default/default/upstream/7365855022044863454 {"id":7365855022044863454,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8f9d0fc-777a-4c62-bf0c-514ac07d9e98 {"id":"e8f9d0fc-777a-4c62-bf0c-514ac07d9e98","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996779} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc5e0fa38 e8f9d0fc-777a-4c62-bf0c-514ac07d9e98 /tidb/cdc/default/default/upstream/7365855022044863454 {"id":7365855022044863454,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449576115287621633 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.30613.out cli changefeed create --start-ts=449576115287621633 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 table multi_tables_ddl_test.finish_mark exists check table exists success + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s + info='{ "upstream_id": 7365854946181955244, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449576116405141508, "checkpoint_time": "2024-05-06 19:59:41.941", "error": null }' + echo '{ "upstream_id": 7365854946181955244, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449576116405141508, "checkpoint_time": "2024-05-06 19:59:41.941", "error": null }' { "upstream_id": 7365854946181955244, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449576116405141508, "checkpoint_time": "2024-05-06 19:59:41.941", "error": null } ++ echo '{' '"upstream_id":' 7365854946181955244, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449576116405141508, '"checkpoint_time":' '"2024-05-06' '19:59:41.941",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365854946181955244, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449576116405141508, '"checkpoint_time":' '"2024-05-06' '19:59:41.941",' '"error":' null '}' ++ jq -r .error.message Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365855022044863454,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:59:42.693290054+08:00","start_ts":449576115287621633,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576115287621633,"checkpoint_ts":449576115287621633,"checkpoint_time":"2024-05-06 19:59:37.678"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7365854946181955244, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449576116405141508, "checkpoint_time": "2024-05-06 19:59:41.941", "error": null }' + echo '{ "upstream_id": 7365854946181955244, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449576116405141508, "checkpoint_time": "2024-05-06 19:59:41.941", "error": null }' { "upstream_id": 7365854946181955244, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449576116405141508, "checkpoint_time": "2024-05-06 19:59:41.941", "error": null } ++ echo '{' '"upstream_id":' 7365854946181955244, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449576116405141508, '"checkpoint_time":' '"2024-05-06' '19:59:41.941",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365854946181955244, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449576116405141508, '"checkpoint_time":' '"2024-05-06' '19:59:41.941",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s + info='{ "upstream_id": 7365854946181955244, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449576112538517541, "checkpoint_time": "2024-05-06 19:59:27.191", "error": { "time": "2024-05-06T19:59:29.99497339+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7365854946181955244, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449576112538517541, "checkpoint_time": "2024-05-06 19:59:27.191", "error": { "time": "2024-05-06T19:59:29.99497339+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7365854946181955244, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449576112538517541, "checkpoint_time": "2024-05-06 19:59:27.191", "error": { "time": "2024-05-06T19:59:29.99497339+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '130', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365854946181955244, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449576112538517541, '"checkpoint_time":' '"2024-05-06' '19:59:27.191",' '"error":' '{' '"time":' '"2024-05-06T19:59:29.99497339+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''130'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365854946181955244, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449576112538517541, '"checkpoint_time":' '"2024-05-06' '19:59:27.191",' '"error":' '{' '"time":' '"2024-05-06T19:59:29.99497339+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''130'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '130', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully + run_case_with_unavailable_tikv conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory wait process cdc.test exit for 1-th time... table consistent_replicate_storage_s3.check1 not exists for 3-th check, retry later + set +x + run_sql 'use test; create table t1(id bigint primary key, a text, b text as ((regexp_replace(a, '\''^[1-9]\d{9,29}$'\'', '\''aaaaa'\''))), c text); insert into t1 (id, a, c) values(1,123456, '\''ab\\\\c'\''); insert into t1 (id, a, c) values(2,1234567890123, '\''ab\\c'\'');' 127.0.0.1 4000 + '[' mysql == mysql ']' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:59:44 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aef8dea5-204b-4f08-9ba1-aa5f71625974 {"id":"aef8dea5-204b-4f08-9ba1-aa5f71625974","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996781} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc5ec0935 aef8dea5-204b-4f08-9ba1-aa5f71625974 /tidb/cdc/default/default/upstream/7365855032029440261 {"id":7365855032029440261,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aef8dea5-204b-4f08-9ba1-aa5f71625974 {"id":"aef8dea5-204b-4f08-9ba1-aa5f71625974","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996781} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc5ec0935 aef8dea5-204b-4f08-9ba1-aa5f71625974 /tidb/cdc/default/default/upstream/7365855032029440261 {"id":7365855032029440261,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aef8dea5-204b-4f08-9ba1-aa5f71625974 {"id":"aef8dea5-204b-4f08-9ba1-aa5f71625974","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996781} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc5ec0935 aef8dea5-204b-4f08-9ba1-aa5f71625974 /tidb/cdc/default/default/upstream/7365855032029440261 {"id":7365855032029440261,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449576115886096385 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.16949.out cli changefeed create --start-ts=449576115886096385 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365855032029440261,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:59:44.981533603+08:00","start_ts":449576115886096385,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576115886096385,"checkpoint_ts":449576115886096385,"checkpoint_time":"2024-05-06 19:59:39.961"} PASS wait process cdc.test exit for 3-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/common_1/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... coverage: 2.7% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:59:45 CST 2024] <<<<<< run test case multi_tables_ddl success! >>>>>> Exiting on signal: INTERRUPT table consistent_replicate_storage_s3.check1 not exists for 4-th check, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... table test.t1 exists + sleep 10 + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table consistent_replicate_storage_s3.check1 not exists for 5-th check, retry later Verifying downstream PD is started... table test.t1 exists + sleep 5 start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... table consistent_replicate_storage_s3.check1 not exists for 6-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table consistent_replicate_storage_s3.check1 not exists for 7-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_s3.check1 not exists for 8-th check, retry later + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 15483 3.7 0.0 2652348 191896 ? Sl 19:59 0:00 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1714996771341561935.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb.log jenkins 15487 12.9 0.0 2736072 241652 ? Sl 19:59 0:02 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1714996771344466877.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb_other.log jenkins 15569 13.5 0.0 2277096 260812 ? Sl 19:59 0:02 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1714996771393033684.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 1871 0 --:--:-- --:--:-- --:--:-- 1883 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:59:52.361","puller_resolved_ts":"2024-05-06 19:59:46.461","last_synced_ts":"2024-05-06 19:59:46.961","now_ts":"2024-05-06 19:59:54.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:59:52.361","puller_resolved_ts":"2024-05-06' '19:59:46.461","last_synced_ts":"2024-05-06' '19:59:46.961","now_ts":"2024-05-06' '19:59:54.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:59:52.361","puller_resolved_ts":"2024-05-06' '19:59:46.461","last_synced_ts":"2024-05-06' '19:59:46.961","now_ts":"2024-05-06' '19:59:54.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_s3.check1 not exists for 9-th check, retry later + run_sql 'SELECT * from test.t1' 127.0.0.1 3306 + check_contains 'b: 123456' + check_contains 'b: aaaaa' + check_contains 'c: ab\\\\c' + check_contains 'c: ab\\c' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + stop_tidb_cluster table consistent_replicate_storage_s3.check1 not exists for 10-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371976f8000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:19573, start at 2024-05-06 19:59:56.875276792 +0800 CST m=+5.146751610 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:56.882 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:56.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:56.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371976f8000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:19573, start at 2024-05-06 19:59:56.875276792 +0800 CST m=+5.146751610 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:56.882 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:56.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:56.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371976d40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:19645, start at 2024-05-06 19:59:56.886738513 +0800 CST m=+5.109621382 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:56.893 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:56.853 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:56.853 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.20931.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // dir table consistent_replicate_storage_s3.check1 not exists for 11-th check, retry later [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + set +x + tso='449576121220988929 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576121220988929 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449576121220988929 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Mon May 6 20:00:01 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2097820980.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_storage_s3.check1 not exists for 12-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_s3.check1 not exists for 13-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 12:00:04 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44182b53-587b-44d0-93e6-e7e9f29d6246 {"id":"44182b53-587b-44d0-93e6-e7e9f29d6246","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996802} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc631c739 44182b53-587b-44d0-93e6-e7e9f29d6246 /tidb/cdc/default/default/upstream/7365855112322559989 {"id":7365855112322559989,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44182b53-587b-44d0-93e6-e7e9f29d6246 {"id":"44182b53-587b-44d0-93e6-e7e9f29d6246","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996802} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc631c739 44182b53-587b-44d0-93e6-e7e9f29d6246 /tidb/cdc/default/default/upstream/7365855112322559989 {"id":7365855112322559989,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44182b53-587b-44d0-93e6-e7e9f29d6246 {"id":"44182b53-587b-44d0-93e6-e7e9f29d6246","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996802} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc631c739 44182b53-587b-44d0-93e6-e7e9f29d6246 /tidb/cdc/default/default/upstream/7365855112322559989 {"id":7365855112322559989,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449576121220988929 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.21035.out cli changefeed create --start-ts=449576121220988929 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365855112322559989,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T20:00:05.299745942+08:00","start_ts":449576121220988929,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576121220988929,"checkpoint_ts":449576121220988929,"checkpoint_time":"2024-05-06 20:00:00.312"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... table consistent_replicate_storage_s3.check1 not exists for 14-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3719ec1c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-fd5nv, pid:6099, start at 2024-05-06 20:00:04.37863438 +0800 CST m=+10.349558167 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:02:04.411 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-20:00:04.410 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:50:04.410 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3719ec1c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-fd5nv, pid:6099, start at 2024-05-06 20:00:04.37863438 +0800 CST m=+10.349558167 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:02:04.411 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-20:00:04.410 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:50:04.410 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3719e9cc000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-fd5nv, pid:6131, start at 2024-05-06 20:00:04.257222794 +0800 CST m=+10.121206732 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:02:04.301 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-20:00:04.311 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:50:04.311 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/common_1/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/common_1/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/common_1/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + check_table_exists test.t1 127.0.0.1 3306 table test.t1 exists + sleep 5 table consistent_replicate_storage_s3.check1 exists table consistent_replicate_storage_s3.t2 not exists for 1-th check, retry later + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.7364.out cli tso query --pd=http://127.0.0.1:2379 table consistent_replicate_storage_s3.t2 not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... + set +x + tso='449576123645296643 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576123645296643 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 20:00:11 CST 2024] <<<<<< START cdc server in common_1 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.73947396.out server --log-file /tmp/tidb_cdc_test/common_1/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/common_1/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_storage_s3.t2 not exists for 3-th check, retry later + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 18993 22.2 0.4 3971552 1869460 ? Sl 19:59 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 18994 21.6 0.4 3965924 1890856 ? Sl 19:59 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 18995 29.0 0.4 4004832 1927732 ? Sl 19:59 0:06 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 18997 28.9 0.4 3993572 1917356 ? Sl 19:59 0:06 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2420 0 --:--:-- --:--:-- --:--:-- 2430 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 20:00:11.762","puller_resolved_ts":"2024-05-06 20:00:06.762","last_synced_ts":"2024-05-06 20:00:07.262","now_ts":"2024-05-06 20:00:12.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '20:00:11.762","puller_resolved_ts":"2024-05-06' '20:00:06.762","last_synced_ts":"2024-05-06' '20:00:07.262","now_ts":"2024-05-06' '20:00:12.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '20:00:11.762","puller_resolved_ts":"2024-05-06' '20:00:06.762","last_synced_ts":"2024-05-06' '20:00:07.262","now_ts":"2024-05-06' '20:00:12.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 table consistent_replicate_storage_s3.t2 not exists for 4-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 12:00:14 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c98e4456-f5bc-47e0-80c7-54e80ef327b6 {"id":"c98e4456-f5bc-47e0-80c7-54e80ef327b6","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996811} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc63e7546 c98e4456-f5bc-47e0-80c7-54e80ef327b6 /tidb/cdc/default/default/upstream/7365855128808024665 {"id":7365855128808024665,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c98e4456-f5bc-47e0-80c7-54e80ef327b6 {"id":"c98e4456-f5bc-47e0-80c7-54e80ef327b6","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996811} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc63e7546 c98e4456-f5bc-47e0-80c7-54e80ef327b6 /tidb/cdc/default/default/upstream/7365855128808024665 {"id":7365855128808024665,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c98e4456-f5bc-47e0-80c7-54e80ef327b6 {"id":"c98e4456-f5bc-47e0-80c7-54e80ef327b6","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996811} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc63e7546 c98e4456-f5bc-47e0-80c7-54e80ef327b6 /tidb/cdc/default/default/upstream/7365855128808024665 {"id":7365855128808024665,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 8828494d-ba43-41f4-8b1b-d011aaaa9a67 Info: {"upstream_id":7365855128808024665,"namespace":"default","id":"8828494d-ba43-41f4-8b1b-d011aaaa9a67","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-06T20:00:14.72330392+08:00","start_ts":449576123645296643,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576123645296643,"checkpoint_ts":449576123645296643,"checkpoint_time":"2024-05-06 20:00:09.560"} table consistent_replicate_storage_s3.t2 not exists for 5-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_s3.t2 not exists for 6-th check, retry later skip some SQLs in tidb v4.0.x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371814880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:29206, start at 2024-05-06 19:59:34.207803495 +0800 CST m=+5.194312681 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:34.214 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:34.178 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:34.178 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371814880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:29206, start at 2024-05-06 19:59:34.207803495 +0800 CST m=+5.194312681 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:34.214 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:34.178 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:34.178 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371815e80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:29271, start at 2024-05-06 19:59:34.309804646 +0800 CST m=+5.230471163 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:34.319 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:34.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:34.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 2 times to try to start tidb cluster... table consistent_replicate_storage_s3.t2 exists check diff successfully table common_1.v1 exists table common_1.recover_and_insert not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 20:00:22 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_s3 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_s3.1779317795.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table common_1.recover_and_insert not exists for 2-th check, retry later table common_1.recover_and_insert not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 12:00:25 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/554b04f0-44cf-4c0f-b47a-e1f39b818775 {UpstreamID:7365854998956661486 Namespace:default ID:554b04f0-44cf-4c0f-b47a-e1f39b818775 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:59:36.783887178 +0800 CST StartTs:449576115016302598 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0017fa000 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576115042516999} {CheckpointTs:449576127533678594 MinTableBarrierTs:449576127533678595 AdminJobType:noop} span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449576127533678595, checkpointTs: 449576127533678594, state: Replicating span: {table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}, resolvedTs: 449576127533678594, checkpointTs: 449576127533678594, state: Replicating span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449576127533678595, checkpointTs: 449576127533678594, state: Replicating span: {table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}, resolvedTs: 449576127533678595, checkpointTs: 449576127533678594, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576127533678595, checkpointTs: 449576127533678594, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449576127533678595, checkpointTs: 449576127533678594, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576127533678594, checkpointTs: 449576127533678594, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449576127533678595, checkpointTs: 449576127533678594, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4678d427-1221-4465-91b5-f62170c54f77 {"id":"4678d427-1221-4465-91b5-f62170c54f77","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996822} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc5c64937 4678d427-1221-4465-91b5-f62170c54f77 /tidb/cdc/default/default/changefeed/info/554b04f0-44cf-4c0f-b47a-e1f39b818775 {"upstream-id":7365854998956661486,"namespace":"default","changefeed-id":"554b04f0-44cf-4c0f-b47a-e1f39b818775","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:59:36.783887178+08:00","start-ts":449576115016302598,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576115042516999} /tidb/cdc/default/default/changefeed/status/554b04f0-44cf-4c0f-b47a-e1f39b818775 {"checkpoint-ts":449576127533678594,"min-table-barrier-ts":449576127795560455,"admin-job-type":0} /tidb/cdc/default/default/task/position/4678d427-1221-4465-91b5-f62170c54f77/554b04f0-44cf-4c0f-b47a-e1f39b818775 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854998956661486 {"id":7365854998956661486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/554b04f0-44cf-4c0f-b47a-e1f39b818775 {UpstreamID:7365854998956661486 Namespace:default ID:554b04f0-44cf-4c0f-b47a-e1f39b818775 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:59:36.783887178 +0800 CST StartTs:449576115016302598 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0017fa000 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576115042516999} {CheckpointTs:449576127533678594 MinTableBarrierTs:449576127533678595 AdminJobType:noop} span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449576127533678595, checkpointTs: 449576127533678594, state: Replicating span: {table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}, resolvedTs: 449576127533678594, checkpointTs: 449576127533678594, state: Replicating span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449576127533678595, checkpointTs: 449576127533678594, state: Replicating span: {table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}, resolvedTs: 449576127533678595, checkpointTs: 449576127533678594, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576127533678595, checkpointTs: 449576127533678594, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449576127533678595, checkpointTs: 449576127533678594, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576127533678594, checkpointTs: 449576127533678594, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449576127533678595, checkpointTs: 449576127533678594, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4678d427-1221-4465-91b5-f62170c54f77 {"id":"4678d427-1221-4465-91b5-f62170c54f77","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996822} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc5c64937 4678d427-1221-4465-91b5-f62170c54f77 /tidb/cdc/default/default/changefeed/info/554b04f0-44cf-4c0f-b47a-e1f39b818775 {"upstream-id":7365854998956661486,"namespace":"default","changefeed-id":"554b04f0-44cf-4c0f-b47a-e1f39b818775","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:59:36.783887178+08:00","start-ts":449576115016302598,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576115042516999} /tidb/cdc/default/default/changefeed/status/554b04f0-44cf-4c0f-b47a-e1f39b818775 {"checkpoint-ts":449576127533678594,"min-table-barrier-ts":449576127795560455,"admin-job-type":0} /tidb/cdc/default/default/task/position/4678d427-1221-4465-91b5-f62170c54f77/554b04f0-44cf-4c0f-b47a-e1f39b818775 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854998956661486 {"id":7365854998956661486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/554b04f0-44cf-4c0f-b47a-e1f39b818775 {UpstreamID:7365854998956661486 Namespace:default ID:554b04f0-44cf-4c0f-b47a-e1f39b818775 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 19:59:36.783887178 +0800 CST StartTs:449576115016302598 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0017fa000 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576115042516999} {CheckpointTs:449576127533678594 MinTableBarrierTs:449576127533678595 AdminJobType:noop} span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449576127533678595, checkpointTs: 449576127533678594, state: Replicating span: {table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}, resolvedTs: 449576127533678594, checkpointTs: 449576127533678594, state: Replicating span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449576127533678595, checkpointTs: 449576127533678594, state: Replicating span: {table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}, resolvedTs: 449576127533678595, checkpointTs: 449576127533678594, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576127533678595, checkpointTs: 449576127533678594, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449576127533678595, checkpointTs: 449576127533678594, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449576127533678594, checkpointTs: 449576127533678594, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449576127533678595, checkpointTs: 449576127533678594, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4678d427-1221-4465-91b5-f62170c54f77 {"id":"4678d427-1221-4465-91b5-f62170c54f77","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996822} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc5c64937 4678d427-1221-4465-91b5-f62170c54f77 /tidb/cdc/default/default/changefeed/info/554b04f0-44cf-4c0f-b47a-e1f39b818775 {"upstream-id":7365854998956661486,"namespace":"default","changefeed-id":"554b04f0-44cf-4c0f-b47a-e1f39b818775","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T19:59:36.783887178+08:00","start-ts":449576115016302598,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576115042516999} /tidb/cdc/default/default/changefeed/status/554b04f0-44cf-4c0f-b47a-e1f39b818775 {"checkpoint-ts":449576127533678594,"min-table-barrier-ts":449576127795560455,"admin-job-type":0} /tidb/cdc/default/default/task/position/4678d427-1221-4465-91b5-f62170c54f77/554b04f0-44cf-4c0f-b47a-e1f39b818775 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365854998956661486 {"id":7365854998956661486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table common_1.recover_and_insert exists table common_1.finish_mark not exists for 1-th check, retry later table common_1.finish_mark not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... table common_1.finish_mark not exists for 3-th check, retry later table common_1.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 20:00:35 CST 2024] <<<<<< run test case common_1 success! >>>>>> Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371814880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:29206, start at 2024-05-06 19:59:34.207803495 +0800 CST m=+5.194312681 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:34.214 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:34.178 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:34.178 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371814880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:29206, start at 2024-05-06 19:59:34.207803495 +0800 CST m=+5.194312681 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:34.214 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:34.178 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:34.178 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371815e80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:29271, start at 2024-05-06 19:59:34.309804646 +0800 CST m=+5.230471163 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:34.319 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:34.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:34.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } check_redo_resolved_ts 554b04f0-44cf-4c0f-b47a-e1f39b818775 449576133418549255 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data/redo/554b04f0-44cf-4c0f-b47a-e1f39b818775/meta global resolved ts 449576132796481539 not forward to 449576133418549255 run task failed 1-th time, retry later [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check_redo_resolved_ts 554b04f0-44cf-4c0f-b47a-e1f39b818775 449576133418549255 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data/redo/554b04f0-44cf-4c0f-b47a-e1f39b818775/meta global resolved ts 449576133313953796 not forward to 449576133418549255 run task failed 2-th time, retry later check_redo_resolved_ts 554b04f0-44cf-4c0f-b47a-e1f39b818775 449576133418549255 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data/redo/554b04f0-44cf-4c0f-b47a-e1f39b818775/meta run task successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [2024/05/06 20:00:55.343 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 20:00:55.344 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2024/05/06 20:00:55.344 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=:6060] [2024/05/06 20:00:55.346 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/06 20:00:55.349 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 20:00:55.349 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449576127887573003] [resolvedTs=449576134886555654] [2024/05/06 20:00:55.350 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/06 20:00:55.350 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 20:00:55.350 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/06 20:00:55.350 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 20:00:55.350 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/06 20:00:55.355 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=5.644127ms] [2024/05/06 20:00:55.355 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 20:00:55.360 +08:00] [INFO] [mysql.go:197] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 20:00:55.361 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:00:55.361 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:00:55.361 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:00:55.361 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:00:55.361 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:00:55.361 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:00:55.361 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:00:55.361 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:00:55.361 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:00:55.361 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:00:55.361 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:00:55.361 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:00:55.361 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:00:55.361 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:00:55.361 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:00:55.361 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:00:55.361 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 20:00:55.361 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 20:00:55.361 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 20:00:55.361 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 20:00:55.361 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 20:00:55.361 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 20:00:55.361 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 20:00:55.361 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 20:00:55.361 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 20:00:55.361 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 20:00:55.361 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 20:00:55.361 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 20:00:55.361 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 20:00:55.361 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 20:00:55.361 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 20:00:55.361 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 20:00:55.361 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 20:00:55.361 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 20:00:55.361 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/06 20:00:55.367 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 20:00:55.370 +08:00] [INFO] [mysql_ddl_sink.go:105] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/06 20:00:55.742 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=392.291382ms] [2024/05/06 20:00:55.742 +08:00] [ERROR] [redo.go:268] ["ignore unsupported DDL"] [ddl="{\"StartTs\":449576127887572997,\"CommitTs\":449576127887573003,\"Query\":\"ALTER TABLE `consistent_replicate_storage_s3`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_s3`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_storage_s3\",\"Table\":\"t1\",\"TableID\":110,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 20:00:56.993 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}] [checkpointTs=449576134886555654] [2024/05/06 20:00:56.993 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}] [checkpointTs=449576134886555654] [2024/05/06 20:00:57.094 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}] [checkpointTs=449576134886555654] [2024/05/06 20:00:57.094 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}] [checkpointTs=449576134886555654] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371814880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:29206, start at 2024-05-06 19:59:34.207803495 +0800 CST m=+5.194312681 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:34.214 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:34.178 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:34.178 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371814880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:29206, start at 2024-05-06 19:59:34.207803495 +0800 CST m=+5.194312681 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:34.214 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:34.178 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:34.178 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371815e80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-g8f72, pid:29271, start at 2024-05-06 19:59:34.309804646 +0800 CST m=+5.230471163 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:01:34.319 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:59:34.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:49:34.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 3306 ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.34388.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 20:00:57.802 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:120,start_key:7480000000000000ff785f720000000000fa,end_key:7480000000000000ff785f730000000000fa}] [checkpointTs=449576134886555654] [2024/05/06 20:00:57.802 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:120,start_key:7480000000000000ff785f720000000000fa,end_key:7480000000000000ff785f730000000000fa}] [checkpointTs=449576134886555654] [2024/05/06 20:00:57.902 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=5008] [appliedDDLCount=0] [currentCheckpoint=449576134886555654] [2024/05/06 20:00:57.902 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 20:00:57.902 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 20:00:57.902 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/06 20:00:57.902 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/06 20:00:57.902 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 20:00:57.902 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/06 20:00:57.902 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 20:00:57.902 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/06 20:00:57.902 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 20:00:57.902 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 20:00:57.902 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 20:00:57.902 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/06 20:00:57.902 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 20:00:57.902 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 20:00:57.902 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/06 20:00:57.902 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] Apply redo log successfully check diff successfully [Mon May 6 20:00:58 CST 2024] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> count(*) 5000 Exiting on signal: INTERRUPT + set +x + tso='449576136223490049 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576136223490049 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449576136223490049 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Mon May 6 20:00:59 CST 2024] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.3442734429.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 12:01:02 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7365855022044863454 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:59:42.693290054 +0800 CST StartTs:449576115287621633 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001387680 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576116598341634} {CheckpointTs:449576119848665095 MinTableBarrierTs:449576120110809095 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8f9d0fc-777a-4c62-bf0c-514ac07d9e98 {"id":"e8f9d0fc-777a-4c62-bf0c-514ac07d9e98","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996779} /tidb/cdc/default/__cdc_meta__/capture/fca8ffc2-0657-4469-8003-185c644c8feb {"id":"fca8ffc2-0657-4469-8003-185c644c8feb","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996859} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc5e0fa38 e8f9d0fc-777a-4c62-bf0c-514ac07d9e98 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc72349d0 fca8ffc2-0657-4469-8003-185c644c8feb /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7365855022044863454,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:59:42.693290054+08:00","start-ts":449576115287621633,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576116598341634} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":449576119848665095,"min-table-barrier-ts":449576120110809095,"admin-job-type":0} /tidb/cdc/default/default/task/position/e8f9d0fc-777a-4c62-bf0c-514ac07d9e98/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/fca8ffc2-0657-4469-8003-185c644c8feb/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365855022044863454 {"id":7365855022044863454,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7365855022044863454 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:59:42.693290054 +0800 CST StartTs:449576115287621633 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001387680 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576116598341634} {CheckpointTs:449576119848665095 MinTableBarrierTs:449576120110809095 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8f9d0fc-777a-4c62-bf0c-514ac07d9e98 {"id":"e8f9d0fc-777a-4c62-bf0c-514ac07d9e98","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996779} /tidb/cdc/default/__cdc_meta__/capture/fca8ffc2-0657-4469-8003-185c644c8feb {"id":"fca8ffc2-0657-4469-8003-185c644c8feb","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996859} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc5e0fa38 e8f9d0fc-777a-4c62-bf0c-514ac07d9e98 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc72349d0 fca8ffc2-0657-4469-8003-185c644c8feb /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7365855022044863454,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:59:42.693290054+08:00","start-ts":449576115287621633,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576116598341634} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":449576119848665095,"min-table-barrier-ts":449576120110809095,"admin-job-type":0} /tidb/cdc/default/default/task/position/e8f9d0fc-777a-4c62-bf0c-514ac07d9e98/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/fca8ffc2-0657-4469-8003-185c644c8feb/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"er+ grep -q 'failed to get info:' ror":null,"warning":null} /tidb/cdc/default/default/upstream/7365855022044863454 {"id":7365855022044863454,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7365855022044863454 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:59:42.693290054 +0800 CST StartTs:449576115287621633 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001387680 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576116598341634} {CheckpointTs:449576119848665095 MinTableBarrierTs:449576120110809095 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8f9d0fc-777a-4c62-bf0c-514ac07d9e98 {"id":"e8f9d0fc-777a-4c62-bf0c-514ac07d9e98","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996779} /tidb/cdc/default/__cdc_meta__/capture/fca8ffc2-0657-4469-8003-185c644c8feb {"id":"fca8ffc2-0657-4469-8003-185c644c8feb","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996859} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc5e0fa38 e8f9d0fc-777a-4c62-bf0c-514ac07d9e98 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc72349d0 fca8ffc2-0657-4469-8003-185c644c8feb /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7365855022044863454,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:59:42.693290054+08:00","start-ts":449576115287621633,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576116598341634} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":449576119848665095,"min-table-barrier-ts":449576120110809095,"admin-job-type":0} /tidb/cdc/default/default/task/position/e8f9d0fc-777a-4c62-bf0c-514ac07d9e98/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/fca8ffc2-0657-4469-8003-185c644c8feb/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365855022044863454 {"id":7365855022044863454,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449576136223490049 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.34474.out cli changefeed create --start-ts=449576136223490049 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 Create changefeed successfully! ID: test-2 Info: {"upstream_id":7365855022044863454,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T20:01:05.049544485+08:00","start_ts":449576136223490049,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576136223490049,"checkpoint_ts":449576136223490049,"checkpoint_time":"2024-05-06 20:00:57.542"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'use test; create table t2(id bigint primary key, a date); insert into t2 values(1, '\''2023-02-08'\'');' 127.0.0.1 4000 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/consistent_partition_table Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371eaf280012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:19167, start at 2024-05-06 20:01:22.394104696 +0800 CST m=+5.179772045 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:03:22.403 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-20:01:22.378 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:51:22.378 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371eaf280012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:19167, start at 2024-05-06 20:01:22.394104696 +0800 CST m=+5.179772045 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:03:22.403 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-20:01:22.378 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:51:22.378 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d371eafd40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-bwx84, pid:19242, start at 2024-05-06 20:01:22.467586792 +0800 CST m=+5.181814352 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:03:22.474 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-20:01:22.470 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:51:22.470 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_partition_table/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_partition_table/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_partition_table/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 20:01:25 CST 2024] <<<<<< START cdc server in consistent_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_partition_table.2053420536.out server --log-file /tmp/tidb_cdc_test/consistent_partition_table/cdcpartition_table.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_partition_table/cdc_datapartition_table.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 12:01:28 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/375bee8c-b2b7-4afc-9a54-b48ad966f4ae {"id":"375bee8c-b2b7-4afc-9a54-b48ad966f4ae","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996885} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc77fd22a 375bee8c-b2b7-4afc-9a54-b48ad966f4ae /tidb/cdc/default/default/upstream/7365855490262725066 {"id":7365855490262725066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/375bee8c-b2b7-4afc-9a54-b48ad966f4ae {"id":"375bee8c-b2b7-4afc-9a54-b48ad966f4ae","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996885} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc77fd22a 375bee8c-b2b7-4afc-9a54-b48ad966f4ae /tidb/cdc/default/default/upstream/7365855490262725066 {"id":7365855490262725066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/375bee8c-b2b7-4afc-9a54-b48ad966f4ae {"id":"375bee8c-b2b7-4afc-9a54-b48ad966f4ae","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996885} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc77fd22a 375bee8c-b2b7-4afc-9a54-b48ad966f4ae /tidb/cdc/default/default/upstream/7365855490262725066 {"id":7365855490262725066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table partition_table2.t2 not exists for 1-th check, retry later table partition_table2.t2 not exists for 2-th check, retry later table partition_table2.t2 not exists for 3-th check, retry later table partition_table2.t2 not exists for 4-th check, retry later table partition_table2.t2 not exists for 5-th check, retry later table partition_table2.t2 not exists for 6-th check, retry later table partition_table2.t2 not exists for 7-th check, retry later table partition_table2.t2 not exists for 8-th check, retry later table partition_table2.t2 not exists for 9-th check, retry later table partition_table2.t2 not exists for 10-th check, retry later table partition_table2.t2 exists wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 20:01:56 CST 2024] <<<<<< START cdc server in consistent_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true);github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_partition_table.2073720739.out server --log-file /tmp/tidb_cdc_test/consistent_partition_table/cdcpartition_table.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_partition_table/cdc_datapartition_table.server2 --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 12:01:59 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/98e95524-f388-4832-b881-155e64adcff0 {UpstreamID:7365855490262725066 Namespace:default ID:98e95524-f388-4832-b881-155e64adcff0 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 20:01:28.8279948 +0800 CST StartTs:449576144398974978 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004198870 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576144412082187} {CheckpointTs:449576150585311234 MinTableBarrierTs:449576152420319237 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449576152158437378, checkpointTs: 449576150585311234, state: Replicating span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449576150585311234, checkpointTs: 449576150585311234, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:107,start_key:7480000000000000ff6b5f720000000000fa,end_key:7480000000000000ff6b5f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:109,start_key:7480000000000000ff6d5f720000000000fa,end_key:7480000000000000ff6d5f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}, resolvedTs: 449576152158437380, checkpointTs: 449576150585311234, state: Replicating span: {table_id:121,start_key:7480000000000000ff795f720000000000fa,end_key:7480000000000000ff795f730000000000fa}, resolvedTs: 449576150585311234, checkpointTs: 449576150585311234, state: Replicating span: {table_id:125,start_key:7480000000000000ff7d5f720000000000fa,end_key:7480000000000000ff7d5f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:119,start_key:7480000000000000ff775f720000000000fa,end_key:7480000000000000ff775f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3de4178a-507f-4be7-b629-cb4db593511c {"id":"3de4178a-507f-4be7-b629-cb4db593511c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996916} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc77fd37e 3de4178a-507f-4be7-b629-cb4db593511c /tidb/cdc/default/default/changefeed/info/98e95524-f388-4832-b881-155e64adcff0 {"upstream-id":7365855490262725066,"namespace":"default","changefeed-id":"98e95524-f388-4832-b881-155e64adcff0","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T20:01:28.8279948+08:00","start-ts":449576144398974978,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576144412082187} /tidb/cdc/default/default/changefeed/status/98e95524-f388-4832-b881-155e64adcff0 {"checkpoint-ts":449576150585311234,"min-table-barrier-ts":449576152420319237,"admin-job-type":0} /tidb/cdc/default/default/task/position/3de4178a-507f-4be7-b629-cb4db593511c/98e95524-f388-4832-b881-155e64adcff0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365855490262725066 {"id":7365855490262725066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/98e95524-f388-4832-b881-155e64adcff0 {UpstreamID:7365855490262725066 Namespace:default ID:98e95524-f388-4832-b881-155e64adcff0 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 20:01:28.8279948 +0800 CST StartTs:449576144398974978 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004198870 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576144412082187} {CheckpointTs:449576150585311234 MinTableBarrierTs:449576152420319237 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449576152158437378, checkpointTs: 449576150585311234, state: Replicating span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449576150585311234, checkpointTs: 449576150585311234, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:107,start_key:7480000000000000ff6b5f720000000000fa,end_key:7480000000000000ff6b5f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:109,start_key:7480000000000000ff6d5f720000000000fa,end_key:7480000000000000ff6d5f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}, resolvedTs: 449576152158437380, checkpointTs: 449576150585311234, state: Replicating span: {table_id:121,start_key:7480000000000000ff795f720000000000fa,end_key:7480000000000000ff795f730000000000fa}, resolvedTs: 449576150585311234, checkpointTs: 449576150585311234, state: Replicating span: {table_id:125,start_key:7480000000000000ff7d5f720000000000fa,end_key:7480000000000000ff7d5f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:119,start_key:7480000000000000ff775f720000000000fa,end_key:7480000000000000ff775f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3de4178a-507f-4be7-b629-cb4db593511c {"id":"3de4178a-507f-4be7-b629-cb4db593511c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996916} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc77fd37e 3de4178a-507f-4be7-b629-cb4db593511c /tidb/cdc/default/default/changefeed/info/98e95524-f388-4832-b881-155e64adcff0 {"upstream-id":7365855490262725066,"namespace":"default","changefeed-id":"98e95524-f388-4832-b881-155e64adcff0","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T20:01:28.8279948+08:00","start-ts":449576144398974978,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576144412082187} /tidb/cdc/default/default/changefeed/status/98e95524-f388-4832-b881-155e64adcff0 {"checkpoint-ts":449576150585311234,"min-table-barrier-ts":449576152420319237,"admin-job-type":0} /tidb/cdc/default/default/task/position/3de4178a-507f-4be7-b629-cb4db593511c/98e95524-f388-4832-b881-155e64adcff0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365855490262725066 {"id":7365855490262725066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/98e95524-f388-4832-b881-155e64adcff0 {UpstreamID:7365855490262725066 Namespace:default ID:98e95524-f388-4832-b881-155e64adcff0 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 20:01:28.8279948 +0800 CST StartTs:449576144398974978 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004198870 State:normal Error: Warning: CreatorVersion:v8.1.0-alpha-23-ge1a27adc6 Epoch:449576144412082187} {CheckpointTs:449576150585311234 MinTableBarrierTs:449576152420319237 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449576152158437378, checkpointTs: 449576150585311234, state: Replicating span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449576150585311234, checkpointTs: 449576150585311234, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:107,start_key:7480000000000000ff6b5f720000000000fa,end_key:7480000000000000ff6b5f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:109,start_key:7480000000000000ff6d5f720000000000fa,end_key:7480000000000000ff6d5f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}, resolvedTs: 449576152158437380, checkpointTs: 449576150585311234, state: Replicating span: {table_id:121,start_key:7480000000000000ff795f720000000000fa,end_key:7480000000000000ff795f730000000000fa}, resolvedTs: 449576150585311234, checkpointTs: 449576150585311234, state: Replicating span: {table_id:125,start_key:7480000000000000ff7d5f720000000000fa,end_key:7480000000000000ff7d5f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating span: {table_id:119,start_key:7480000000000000ff775f720000000000fa,end_key:7480000000000000ff775f730000000000fa}, resolvedTs: 449576152158437381, checkpointTs: 449576150585311234, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3de4178a-507f-4be7-b629-cb4db593511c {"id":"3de4178a-507f-4be7-b629-cb4db593511c","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996916} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc77fd37e 3de4178a-507f-4be7-b629-cb4db593511c /tidb/cdc/default/default/changefeed/info/98e95524-f388-4832-b881-155e64adcff0 {"upstream-id":7365855490262725066,"namespace":"default","changefeed-id":"98e95524-f388-4832-b881-155e64adcff0","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T20:01:28.8279948+08:00","start-ts":449576144398974978,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.1.0-alpha-23-ge1a27adc6","epoch":449576144412082187} /tidb/cdc/default/default/changefeed/status/98e95524-f388-4832-b881-155e64adcff0 {"checkpoint-ts":449576150585311234,"min-table-barrier-ts":449576152420319237,"admin-job-type":0} /tidb/cdc/default/default/task/position/3de4178a-507f-4be7-b629-cb4db593511c/98e95524-f388-4832-b881-155e64adcff0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365855490262725066 {"id":7365855490262725066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + '[' mysql == mysql ']' + check_table_exists test.t2 127.0.0.1 3306 table test.t2 not exists for 1-th check, retry later table test.t2 exists + sleep 10 + run_sql 'SELECT * from test.t2' 127.0.0.1 3306 + check_contains 'a: 2023-02-08' + stop_tidb_cluster ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2569 0 --:--:-- --:--:-- --:--:-- 2569 100 221 100 221 0 0 2565 0 --:--:-- --:--:-- --:--:-- 2540 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 20:02:02.511","puller_resolved_ts":"2024-05-06 20:01:56.510","last_synced_ts":"2024-05-06 19:59:46.961","now_ts":"2024-05-06 20:02:04.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '20:02:02.511","puller_resolved_ts":"2024-05-06' '20:01:56.510","last_synced_ts":"2024-05-06' '19:59:46.961","now_ts":"2024-05-06' '20:02:04.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '20:02:02.511","puller_resolved_ts":"2024-05-06' '20:01:56.510","last_synced_ts":"2024-05-06' '19:59:46.961","now_ts":"2024-05-06' '20:02:04.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... + stop_tidb_cluster Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3722e4e0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:18101, start at 2024-05-06 20:02:31.366048077 +0800 CST m=+7.914129376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:04:31.373 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-20:02:31.352 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:52:31.352 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3722e4e0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:18101, start at 2024-05-06 20:02:31.366048077 +0800 CST m=+7.914129376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:04:31.373 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-20:02:31.352 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:52:31.352 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3722e4c40005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-sfnq4, pid:18188, start at 2024-05-06 20:02:31.350740243 +0800 CST m=+7.846261613 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:04:31.360 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-20:02:31.345 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:52:31.345 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 8902 0 --:--:-- --:--:-- --:--:-- 8925 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 20:00:11.762","puller_resolved_ts":"2024-05-06 20:00:11.813","last_synced_ts":"2024-05-06 20:00:07.262","now_ts":"2024-05-06 20:02:22.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '20:00:11.762","puller_resolved_ts":"2024-05-06' '20:00:11.813","last_synced_ts":"2024-05-06' '20:00:07.262","now_ts":"2024-05-06' '20:02:22.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '20:00:11.762","puller_resolved_ts":"2024-05-06' '20:00:11.813","last_synced_ts":"2024-05-06' '20:00:07.262","now_ts":"2024-05-06' '20:02:22.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... + cd /tmp/tidb_cdc_test/synced_status_with_redo + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.19507.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449576162060402689 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576162060402689 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449576162060402689 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Mon May 6 20:02:37 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1954119543.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 12:02:40 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa9ac3bf-c722-46bd-80f5-6c2ce11429ce {"id":"fa9ac3bf-c722-46bd-80f5-6c2ce11429ce","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996958} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc882823d fa9ac3bf-c722-46bd-80f5-6c2ce11429ce /tidb/cdc/default/default/upstream/7365855770333041588 {"id":7365855770333041588,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa9ac3bf-c722-46bd-80f5-6c2ce11429ce {"id":"fa9ac3bf-c722-46bd-80f5-6c2ce11429ce","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996958} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc882823d fa9ac3bf-c722-46bd-80f5-6c2ce11429ce /tidb/cdc/default/default/upstream/7365855770333041588 {"id":7365855770333041588,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa9ac3bf-c722-46bd-80f5-6c2ce11429ce {"id":"fa9ac3bf-c722-46bd-80f5-6c2ce11429ce","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996958} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc882823d fa9ac3bf-c722-46bd-80f5-6c2ce11429ce /tidb/cdc/default/default/upstream/7365855770333041588 {"id":7365855770333041588,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449576162060402689 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.19597.out cli changefeed create --start-ts=449576162060402689 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365855770333041588,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T20:02:41.160563239+08:00","start_ts":449576162060402689,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576162060402689,"checkpoint_ts":449576162060402689,"checkpoint_time":"2024-05-06 20:02:36.102"} PASS coverage: 2.7% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + sleep 20 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3723c0400018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:22223, start at 2024-05-06 20:02:45.436595077 +0800 CST m=+5.175161982 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:04:45.443 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-20:02:45.442 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:52:45.442 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3723c0400018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:22223, start at 2024-05-06 20:02:45.436595077 +0800 CST m=+5.175161982 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:04:45.443 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-20:02:45.442 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:52:45.442 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3723c1a40009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:22308, start at 2024-05-06 20:02:45.495008802 +0800 CST m=+5.185959555 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:04:45.501 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-20:02:45.481 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:52:45.481 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.23545.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449576165413486593 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576165413486593 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449576165413486593 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Mon May 6 20:02:50 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2358523587.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 12:02:53 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e6306fd8-66b9-401b-9d32-e5e584136f55 {"id":"e6306fd8-66b9-401b-9d32-e5e584136f55","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996970} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc8c80034 e6306fd8-66b9-401b-9d32-e5e584136f55 /tidb/cdc/default/default/upstream/7365855843616750026 {"id":7365855843616750026,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e6306fd8-66b9-401b-9d32-e5e584136f55 {"id":"e6306fd8-66b9-401b-9d32-e5e584136f55","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996970} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc8c80034 e6306fd8-66b9-401b-9d32-e5e584136f55 /tidb/cdc/default/default/upstream/7365855843616750026 {"id":7365855843616750026,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e6306fd8-66b9-401b-9d32-e5e584136f55 {"id":"e6306fd8-66b9-401b-9d32-e5e584136f55","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714996970} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dc8c80034 e6306fd8-66b9-401b-9d32-e5e584136f55 /tidb/cdc/default/default/upstream/7365855843616750026 {"id":7365855843616750026,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449576165413486593 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.23640.out cli changefeed create --start-ts=449576165413486593 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365855843616750026,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T20:02:53.883570909+08:00","start_ts":449576165413486593,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576165413486593,"checkpoint_ts":449576165413486593,"checkpoint_time":"2024-05-06 20:02:48.893"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 22219 3.6 0.0 2661312 192632 ? Sl 20:02 0:00 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714996960251946734.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log jenkins 22223 12.1 0.0 3114416 230772 ? Sl 20:02 0:02 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714996960255400590.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log jenkins 22308 12.7 0.0 2540668 271900 ? Sl 20:02 0:02 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714996960302891639.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 3285 0 --:--:-- --:--:-- --:--:-- 3328 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 20:03:02.292","puller_resolved_ts":"2024-05-06 20:02:55.343","last_synced_ts":"2024-05-06 20:02:55.843","now_ts":"2024-05-06 20:03:02.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '20:03:02.292","puller_resolved_ts":"2024-05-06' '20:02:55.343","last_synced_ts":"2024-05-06' '20:02:55.843","now_ts":"2024-05-06' '20:03:02.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '20:03:02.292","puller_resolved_ts":"2024-05-06' '20:02:55.343","last_synced_ts":"2024-05-06' '20:02:55.843","now_ts":"2024-05-06' '20:03:02.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 8549 0 --:--:-- --:--:-- --:--:-- 8607 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 20:02:36.102","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-06 20:03:02.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '20:02:36.102","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '20:03:02.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '20:02:36.102","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '20:03:02.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + check_logs /tmp/tidb_cdc_test/synced_status_with_redo ++ date + echo '[Mon May 6 20:03:14 CST 2024] <<<<<< run test case synced_status_with_redo success! >>>>>>' [Mon May 6 20:03:14 CST 2024] <<<<<< run test case synced_status_with_redo success! >>>>>> + stop_tidb_cluster \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/partition_table/sync_diff/output" source-instances = ["mysql1"] target-instance = "tidb0" target-check-tables = ["partition_table.?*"] [data-sources] [data-sources.mysql1] host = "127.0.0.1" port = 4000 user = "root" password = "" [data-sources.tidb0] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/05/06 20:04:04.050 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 20:04:04.051 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2024/05/06 20:04:04.051 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=:6060] [2024/05/06 20:04:04.051 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 20:04:04.051 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449576152420319237] [resolvedTs=449576184166219782] [2024/05/06 20:04:04.053 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=1.422475ms] [2024/05/06 20:04:04.053 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=1.524405ms] [2024/05/06 20:04:04.053 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 20:04:04.053 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 20:04:04.053 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/06 20:04:04.059 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 20:04:04.062 +08:00] [INFO] [mysql.go:197] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 20:04:04.063 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:04:04.063 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:04:04.063 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:04:04.063 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:04:04.063 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:04:04.063 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:04:04.063 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:04:04.063 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:04:04.063 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:04:04.063 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:04:04.063 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:04:04.063 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:04:04.063 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:04:04.063 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:04:04.063 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:04:04.063 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 20:04:04.063 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 20:04:04.063 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 20:04:04.063 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 20:04:04.063 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 20:04:04.063 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 20:04:04.063 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 20:04:04.063 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 20:04:04.063 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 20:04:04.063 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 20:04:04.063 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 20:04:04.063 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 20:04:04.063 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 20:04:04.063 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 20:04:04.063 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 20:04:04.063 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 20:04:04.063 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 20:04:04.063 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 20:04:04.063 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 20:04:04.063 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/06 20:04:04.068 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 20:04:04.070 +08:00] [INFO] [mysql_ddl_sink.go:105] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/06 20:04:04.071 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449576152564498438,\"CommitTs\":449576152564498443,\"Query\":\"alter table t1 drop partition p1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"partition_table\",\"Table\":\"t1\",\"TableID\":113,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":20,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 20:04:06.079 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="alter table t1 drop partition p1"] [commitTs=449576152564498443] [namespace=] [changefeed=] [2024/05/06 20:04:06.285 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="alter table t1 drop partition p1"] [duration=206.491888ms] [namespace=] [changefeed=] [2024/05/06 20:04:06.285 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449576152590974986,\"CommitTs\":449576152604082181,\"Query\":\"ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"partition_table\",\"Table\":\"t1\",\"TableID\":113,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 20:04:06.486 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table`.`t2`"] [commitTs=449576152604082181] [namespace=] [changefeed=] [2024/05/06 20:04:06.631 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table`.`t2`"] [duration=144.763699ms] [namespace=] [changefeed=] [2024/05/06 20:04:06.631 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449576152630296585,\"CommitTs\":449576152630296591,\"Query\":\"ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table2`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"partition_table\",\"Table\":\"t1\",\"TableID\":113,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 20:04:07.032 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table2`.`t2`"] [commitTs=449576152630296591] [namespace=] [changefeed=] [2024/05/06 20:04:07.175 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table2`.`t2`"] [duration=142.391288ms] [namespace=] [changefeed=] [2024/05/06 20:04:07.175 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449576153049464840,\"CommitTs\":449576153049464845,\"Query\":\"ALTER TABLE t1 REORGANIZE PARTITION p0,p2 INTO (PARTITION p0 VALUES LESS THAN (5), PARTITION p1 VALUES LESS THAN (10), PARTITION p2 VALUES LESS THAN (21))\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"partition_table\",\"Table\":\"t1\",\"TableID\":113,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":64,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 20:04:07.576 +08:00] [INFO] [mysql_ddl_sink.go:192] ["Start exec DDL"] [DDL="ALTER TABLE t1 REORGANIZE PARTITION p0,p2 INTO (PARTITION p0 VALUES LESS THAN (5), PARTITION p1 VALUES LESS THAN (10), PARTITION p2 VALUES LESS THAN (21))"] [commitTs=449576153049464845] [namespace=] [changefeed=] [2024/05/06 20:04:08.358 +08:00] [INFO] [mysql_ddl_sink.go:239] ["Exec DDL succeeded"] [sql="ALTER TABLE t1 REORGANIZE PARTITION p0,p2 INTO (PARTITION p0 VALUES LESS THAN (5), PARTITION p1 VALUES LESS THAN (10), PARTITION p2 VALUES LESS THAN (21))"] [duration=782.210839ms] [namespace=] [changefeed=] [2024/05/06 20:04:08.358 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:08.358 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:08.458 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:08.458 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:08.559 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:08.559 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:08.659 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:08.659 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:08.759 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:119,start_key:7480000000000000ff775f720000000000fa,end_key:7480000000000000ff775f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:08.759 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:119,start_key:7480000000000000ff775f720000000000fa,end_key:7480000000000000ff775f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:08.859 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:08.859 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:08.960 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:109,start_key:7480000000000000ff6d5f720000000000fa,end_key:7480000000000000ff6d5f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:08.960 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:109,start_key:7480000000000000ff6d5f720000000000fa,end_key:7480000000000000ff6d5f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:09.060 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:09.060 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:09.160 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:121,start_key:7480000000000000ff795f720000000000fa,end_key:7480000000000000ff795f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:09.160 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:121,start_key:7480000000000000ff795f720000000000fa,end_key:7480000000000000ff795f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:09.260 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:107,start_key:7480000000000000ff6b5f720000000000fa,end_key:7480000000000000ff6b5f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:09.260 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:107,start_key:7480000000000000ff6b5f720000000000fa,end_key:7480000000000000ff6b5f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:09.361 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:09.361 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:09.461 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:125,start_key:7480000000000000ff7d5f720000000000fa,end_key:7480000000000000ff7d5f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:09.461 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:125,start_key:7480000000000000ff7d5f720000000000fa,end_key:7480000000000000ff7d5f730000000000fa}] [checkpointTs=449576184166219782] [2024/05/06 20:04:09.561 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=47] [appliedDDLCount=4] [currentCheckpoint=449576184166219782] [2024/05/06 20:04:09.561 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 20:04:09.561 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 20:04:09.561 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 20:04:09.561 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/06 20:04:09.561 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 20:04:09.561 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 20:04:09.561 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/06 20:04:09.561 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 20:04:09.561 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 20:04:09.561 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/06 20:04:09.561 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/06 20:04:09.561 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 20:04:09.561 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/06 20:04:09.561 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 20:04:09.561 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 20:04:09.561 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] Apply redo log successfully check diff successfully [Mon May 6 20:04:09 CST 2024] <<<<<< run test case consistent_partition_table success! >>>>>> Database INFORMATION_SCHEMA METRICS_SCHEMA PERFORMANCE_SCHEMA mysql partition_table partition_table2 sys test =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_big_messages_v2/run.sh using Sink-Type: mysql... <<================= [Mon May 6 20:04:18 CST 2024] <<<<<< run test case kafka_big_messages_v2 success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_tables_ddl_v2/run.sh using Sink-Type: mysql... <<================= [Mon May 6 20:04:22 CST 2024] <<<<<< run test case multi_tables_ddl_v2 success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh using Sink-Type: mysql... <<================= [Mon May 6 20:04:25 CST 2024] <<<<<< run test case multi_topics_v2 success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_cleanup/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_cleanup ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_cleanup + source /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup + CDC_BINARY=cdc.test + SINK_TYPE=mysql + EXIST_FILES=() + CLEANED_FILES=() + trap stop_tidb_cluster EXIT + run mysql + '[' mysql '!=' storage ']' + return + check_logs /tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Mon May 6 20:04:28 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>>' [Mon May 6 20:04:28 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>> + stop_tidb_cluster =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: mysql... <<================= [Mon May 6 20:04:31 CST 2024] <<<<<< run test case csv_storage_basic success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: mysql... <<================= [Mon May 6 20:04:34 CST 2024] <<<<<< run test case csv_storage_multi_tables_ddl success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/csv_storage_partition_table/run.sh using Sink-Type: mysql... <<================= [Mon May 6 20:04:37 CST 2024] <<<<<< run test case csv_storage_partition_table success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2668 0 --:--:-- --:--:-- --:--:-- 2695 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 20:05:12.443","puller_resolved_ts":"2024-05-06 20:05:04.442","last_synced_ts":"2024-05-06 20:02:55.843","now_ts":"2024-05-06 20:05:13.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '20:05:12.443","puller_resolved_ts":"2024-05-06' '20:05:04.442","last_synced_ts":"2024-05-06' '20:02:55.843","now_ts":"2024-05-06' '20:05:13.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '20:05:12.443","puller_resolved_ts":"2024-05-06' '20:05:04.442","last_synced_ts":"2024-05-06' '20:02:55.843","now_ts":"2024-05-06' '20:05:13.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Starting Downstream PD... Release Version: v8.1.0-alpha-20-g13eaf8aaa Edition: Community Git Commit Hash: 13eaf8aaa661dc0d3885ac9e59145f69752cee2d Git Branch: release-8.1 UTC Build Time: 2024-04-30 07:22:55 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.1.0 Edition: Community Git Commit Hash: 56613f7c3e28c02853cc51d15bc1b77f68b58be8 Git Commit Branch: release-8.1 UTC Build Time: 2024-04-30 06:15:40 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.1.0-alpha-106-gc8e6662d10 Edition: Community Git Commit Hash: c8e6662d10deed030ef864169ae27718f5a61991 Git Branch: release-8.1 UTC Build Time: 2024-05-06 10:39:26 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d372e396c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:24789, start at 2024-05-06 20:05:37.014892946 +0800 CST m=+5.192833793 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:07:37.023 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-20:05:36.987 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:55:36.987 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d372e396c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:24789, start at 2024-05-06 20:05:37.014892946 +0800 CST m=+5.192833793 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:07:37.023 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-20:05:36.987 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:55:36.987 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d372e3a240012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:cap-tiflow-release-8-1-pull-cdc-integration-mysql-test-77-3xdmh, pid:24861, start at 2024-05-06 20:05:37.050163143 +0800 CST m=+5.169596397 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-20:07:37.064 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-20:05:37.033 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:55:37.033 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.1.0-alpha-20-geb96e7201 Edition: Community Git Commit Hash: eb96e720183ea798458afc99cfb034864ed75c4c Git Branch: HEAD UTC Build Time: 2024-05-05 06:18:35 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-05 06:22:21 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["eb96e720183ea798458afc99cfb034864ed75c4c"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.1.0-alpha-20-geb96e7201"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.26152.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449576210395824129 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449576210395824129 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449576210395824129 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Mon May 6 20:05:41 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2619326195.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 12:05:45 GMT < Content-Length: 827 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c60554c1-6f83-4088-a455-080297d1ee65 {"id":"c60554c1-6f83-4088-a455-080297d1ee65","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714997142} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dcb626435 c60554c1-6f83-4088-a455-080297d1ee65 /tidb/cdc/default/default/upstream/7365856580744457356 {"id":7365856580744457356,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c60554c1-6f83-4088-a455-080297d1ee65 {"id":"c60554c1-6f83-4088-a455-080297d1ee65","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714997142} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dcb626435 c60554c1-6f83-4088-a455-080297d1ee65 /tidb/cdc/default/default/upstream/7365856580744457356 {"id":7365856580744457356,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c60554c1-6f83-4088-a455-080297d1ee65 {"id":"c60554c1-6f83-4088-a455-080297d1ee65","address":"127.0.0.1:8300","version":"v8.1.0-alpha-23-ge1a27adc6","git-hash":"e1a27adc69acc9d50db0c3fea2945bbd6d983e30","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1714997142} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4dcb626435 c60554c1-6f83-4088-a455-080297d1ee65 /tidb/cdc/default/default/upstream/7365856580744457356 {"id":7365856580744457356,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449576210395824129 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.26248.out cli changefeed create --start-ts=449576210395824129 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-8.1/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365856580744457356,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T20:05:45.474150717+08:00","start_ts":449576210395824129,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.1.0-alpha-23-ge1a27adc6","resolved_ts":449576210395824129,"checkpoint_ts":449576210395824129,"checkpoint_time":"2024-05-06 20:05:40.487"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 11796 0 --:--:-- --:--:-- --:--:-- 11852 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 20:05:40.487","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-06 20:06:06.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '20:05:40.487","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '20:06:06.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '20:05:40.487","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '20:06:06.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + check_logs /tmp/tidb_cdc_test/synced_status ++ date + echo '[Mon May 6 20:06:18 CST 2024] <<<<<< run test case synced_status success! >>>>>>' [Mon May 6 20:06:18 CST 2024] <<<<<< run test case synced_status success! >>>>>> + stop_tidb_cluster \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-8.1-pull_cdc_integration_mysql_test-77/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline Finished: SUCCESS