Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/latest/pull_cdc_integration_storage_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-79k-1pnsk Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-79k-1pnsk is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-79k5x-qbwhw --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2805e3542b857b6a36fdf50eced477094c63faff" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-79k5x" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-79k-1pnsk" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-79k-1pnsk" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-79k-1pnsk" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-79k-1pnsk in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-list --no-walk 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=aa7c97ec-dedc-47cf-8910-cc8dd1787a16 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-79k-1pnsk HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-79k5x GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-ec41a9ea2975eb575c6c655de75271b1a69674254216af41486e1697505d3c9f NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1816-79k5x pingcap-tiflow-pull-cdc-integration-storage-test-1816-79k-1pnsk GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-79k-1pnsk GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.6' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build3850418991=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-storage-test-1816-79k-1pnsk bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-storage-test-1816-79k-1pnsk bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 44353 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: bba42417d2f0cb2a (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 26 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Mon May 06 10:53:32 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11024/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11024/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11024/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11024/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [examples/java/pom.xml] [Pipeline] echo diff file not matched: examples/java/pom.xml [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-c710066-880c493) 203906048 bytes in 0.79 secs (258973641 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/.git/ .git HEAD is now at 129b0926c Merge commit '880c4938eb45789c9feb7606b94fbf04e0a60b7a' into HEAD POST git-upload-pack (656 bytes) From https://github.com/pingcap/tiflow = [up to date] master -> origin/master = [up to date] refs/pull/11024/head -> origin/pr/11024/head Warning: you are leaving 1 commit behind, not connected to any of your branches: 129b0926c Merge commit '880c4938eb45789c9feb7606b94fbf04e0a60b7a' into HEAD If you want to keep it by creating a new branch, this may be a good time to do so with: git branch 129b0926c HEAD is now at c710066a5 *(ticdc): split old update kv entry after restarting changefeed (#10919) ๐Ÿšง Checkouting to base SHA:c710066a51e20310bd1ae34af39296cecf567182... HEAD is now at c710066a5 *(ticdc): split old update kv entry after restarting changefeed (#10919) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: c710066a51e20310bd1ae34af39296cecf567182 c710066a5 *(ticdc): split old update kv entry after restarting changefeed (#10919) be1553484 codec(ticdc): avro simplify the unit test (#11010) 2a7a65c6f Support Sequences (#10203) ๐Ÿšง Pre-merge heads of pull requests to base SHA: c710066a51e20310bd1ae34af39296cecf567182 ... Merge made by the 'ort' strategy. examples/java/pom.xml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) ๐Ÿงพ Pre-merged result: 3c107706d2612c5f422cde6480fd92425795842e 3c107706d Merge commit '880c4938eb45789c9feb7606b94fbf04e0a60b7a' into HEAD c710066a5 *(ticdc): split old update kv entry after restarting changefeed (#10919) 880c4938e Merge branch 'pingcap:master' into fix-java-example-license-issue โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-c710066-880c493 already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./scripts/download-integration-test-binaries.sh master Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 888 0 --:--:-- --:--:-- --:--:-- 891 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 2324 0 --:--:-- --:--:-- --:--:-- 2411 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1988 0 --:--:-- --:--:-- --:--:-- 2050 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 2557 0 --:--:-- --:--:-- --:--:-- 2733 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1/centos7/tidb-server.tar.gz 2024-05-06 18:53:56 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1/centos7/tidb-server.tar.gz [536591132/536591132] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/f83febabecb98b95b098fd31a3664178f8a6b437/centos7/pd-server.tar.gz 2024-05-06 18:54:10 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/f83febabecb98b95b098fd31a3664178f8a6b437/centos7/pd-server.tar.gz [187381047/187381047] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/88099c95a3c0c13a827c0795c3d45070264665e4/centos7/tikv-server.tar.gz 2024-05-06 18:54:50 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/88099c95a3c0c13a827c0795c3d45070264665e4/centos7/tikv-server.tar.gz [919298452/919298452] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e50de84e6d6ecdcc108990217b70b6bb3f50271/centos7/tiflash.tar.gz 2024-05-06 18:55:16 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e50de84e6d6ecdcc108990217b70b6bb3f50271/centos7/tiflash.tar.gz [456051121/456051121] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-06 18:55:21 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-05-06 18:55:24 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-05-06 18:55:24 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-05-06 18:55:25 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz 2024-05-06 18:55:30 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz [79877126/79877126] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> download schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2024-05-06 18:55:46 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.9G drwxr-sr-x 6 jenkins jenkins 4.0K May 6 18:55 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 18:55 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 6 18:55 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 6 18:55 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx 1 jenkins jenkins 13 May 6 12:44 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 May 6 12:44 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 May 6 12:44 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 May 6 12:44 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M May 6 12:05 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 272M May 6 12:45 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 37M May 6 16:50 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M May 6 16:49 pd-ctl -rwxr-xr-x 1 jenkins jenkins 36M May 6 16:50 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 32M May 6 16:49 pd-recover -rwxr-xr-x 1 jenkins jenkins 106M May 6 16:49 pd-server -rwxr-xr-x 1 jenkins jenkins 26M May 6 16:50 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 3.0M May 6 16:50 pd-ut -rwxr-xr-x 1 jenkins jenkins 32M May 6 16:50 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x 1 jenkins jenkins 32M May 6 16:50 stores-dump -rwxr-xr-x 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 208M May 6 18:31 tidb-server -rwxr-xr-x 1 jenkins jenkins 380M May 6 12:44 tiflash -rwxr-xr-x 1 jenkins jenkins 418M May 6 15:33 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.0M May 6 16:50 xprog + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/pd-ut ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ../tiflow/bin/xprog ./bin/ + ls -alh ./bin total 1.9G drwxr-sr-x 6 jenkins jenkins 4.0K May 6 18:55 . drwxr-sr-x 3 jenkins jenkins 4.0K May 6 18:55 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 6 18:55 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 6 18:55 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx 1 jenkins jenkins 13 May 6 12:44 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 May 6 12:44 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 May 6 12:44 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 May 6 12:44 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M May 6 12:05 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 272M May 6 12:45 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 37M May 6 16:50 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M May 6 16:49 pd-ctl -rwxr-xr-x 1 jenkins jenkins 36M May 6 16:50 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 32M May 6 16:49 pd-recover -rwxr-xr-x 1 jenkins jenkins 106M May 6 16:49 pd-server -rwxr-xr-x 1 jenkins jenkins 26M May 6 16:50 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 3.0M May 6 16:50 pd-ut -rwxr-xr-x 1 jenkins jenkins 32M May 6 16:50 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x 1 jenkins jenkins 32M May 6 16:50 stores-dump -rwxr-xr-x 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 208M May 6 18:31 tidb-server -rwxr-xr-x 1 jenkins jenkins 380M May 6 12:44 tiflash -rwxr-xr-x 1 jenkins jenkins 418M May 6 15:33 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.0M May 6 16:50 xprog + ./bin/tidb-server -V Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 + ./bin/tikv-server -V TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.4.0 Git Commit Hash: d671b0840063bc2532941f02e02e12627402844c Git Branch: heads/refs/tags/v7.4.0 UTC Build Time: 2023-09-22 03:51:56 Go Version: go1.21.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x 2 jenkins jenkins 4.0K May 6 18:55 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 18:55 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-25-g3c107706d" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 10:55:50" -X "github.com/pingcap/tiflow/pkg/version.GitHash=3c107706d2612c5f422cde6480fd92425795842e" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-25-g3c107706d"' -o bin/cdc ./cmd/cdc go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading google.golang.org/grpc v1.62.1 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/google/uuid v1.6.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/YangKeao/seahash v0.0.0-20240229041150-e7bf269c3140 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/goccy/go-json v0.10.2 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.33.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/swaggo/swag v1.16.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/docker/go-units v0.5.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/go-openapi/spec v0.21.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/glebarez/sqlite v1.7.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading gorm.io/driver/mysql v1.3.3 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/go-openapi/jsonpointer v0.21.0 go: downloading github.com/go-openapi/jsonreference v0.21.0 go: downloading github.com/go-openapi/swag v0.23.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/glebarez/go-sqlite v1.21.2 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading modernc.org/libc v1.37.1 go: downloading modernc.org/sqlite v1.27.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading k8s.io/apimachinery v0.28.6 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading modernc.org/mathutil v1.6.0 go: downloading modernc.org/memory v1.7.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + '[' -f ./bin/cdc_kafka_consumer ']' + make kafka_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-25-g3c107706d" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 10:58:47" -X "github.com/pingcap/tiflow/pkg/version.GitHash=3c107706d2612c5f422cde6480fd92425795842e" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-25-g3c107706d"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go + '[' -f ./bin/cdc_storage_consumer ']' + make storage_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-25-g3c107706d" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 10:58:54" -X "github.com/pingcap/tiflow/pkg/version.GitHash=3c107706d2612c5f422cde6480fd92425795842e" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-25-g3c107706d"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-25-g3c107706d" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 10:59:02" -X "github.com/pingcap/tiflow/pkg/version.GitHash=3c107706d2612c5f422cde6480fd92425795842e" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-25-g3c107706d"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-25-g3c107706d" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 10:59:02" -X "github.com/pingcap/tiflow/pkg/version.GitHash=3c107706d2612c5f422cde6480fd92425795842e" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-25-g3c107706d"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-25-g3c107706d" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 10:59:02" -X "github.com/pingcap/tiflow/pkg/version.GitHash=3c107706d2612c5f422cde6480fd92425795842e" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-25-g3c107706d"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-25-g3c107706d" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 10:59:02" -X "github.com/pingcap/tiflow/pkg/version.GitHash=3c107706d2612c5f422cde6480fd92425795842e" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-25-g3c107706d"' -o bin/oauth2-server ./cmd/oauth2-server/main.go go: downloading github.com/go-oauth2/oauth2/v4 v4.5.2 go: downloading github.com/tidwall/buntdb v1.3.0 go: downloading github.com/tidwall/rtred v0.1.2 go: downloading github.com/tidwall/grect v0.1.4 go: downloading github.com/tidwall/match v1.1.1 go: downloading github.com/tidwall/gjson v1.14.3 go: downloading github.com/tidwall/tinyqueue v0.1.1 go: downloading github.com/tidwall/pretty v1.2.0 $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20231213065948-336e064b488d go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading go.uber.org/dig v1.13.0 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading go.uber.org/goleak v1.3.0 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-25-g3c107706d" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 10:59:02" -X "github.com/pingcap/tiflow/pkg/version.GitHash=3c107706d2612c5f422cde6480fd92425795842e" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-25-g3c107706d"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-25-g3c107706d" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 10:59:02" -X "github.com/pingcap/tiflow/pkg/version.GitHash=3c107706d2612c5f422cde6480fd92425795842e" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-25-g3c107706d"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.2G drwxr-sr-x 2 jenkins jenkins 4.0K May 6 19:04 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 18:55 .. -rwxr-xr-x 1 jenkins jenkins 220M May 6 19:04 cdc -rwxr-xr-x 1 jenkins jenkins 363M May 6 19:03 cdc.test -rwxr-xr-x 1 jenkins jenkins 183M May 6 18:59 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 182M May 6 18:59 cdc_pulsar_consumer -rwxr-xr-x 1 jenkins jenkins 182M May 6 18:59 cdc_storage_consumer -rwxr-xr-x 1 jenkins jenkins 12M May 6 18:59 oauth2-server + ./bin/cdc version Release Version: v8.2.0-alpha-25-g3c107706d Git Commit Hash: 3c107706d2612c5f422cde6480fd92425795842e Git Branch: HEAD UTC Build Time: 2024-05-06 10:59:02 Go Version: go version go1.21.6 linux/amd64 Failpoint Build: true [Pipeline] } Cache not saved (binary/pingcap/tiflow/cdc-integration-test/rev-c710066-880c493 already exists) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 3.0G drwxr-sr-x 6 jenkins jenkins 4.0K May 6 19:04 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 18:55 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 6 19:04 bin -rwxr-xr-x 1 jenkins jenkins 220M May 6 19:04 cdc -rwxr-xr-x 1 jenkins jenkins 363M May 6 19:03 cdc.test -rwxr-xr-x 1 jenkins jenkins 183M May 6 18:59 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 182M May 6 18:59 cdc_pulsar_consumer -rwxr-xr-x 1 jenkins jenkins 182M May 6 18:59 cdc_storage_consumer drwxr-sr-x 4 jenkins jenkins 4.0K May 6 19:04 etc -rwxr-xr-x 1 jenkins jenkins 17M May 6 19:04 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 6 19:04 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 6 19:04 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 6 19:04 lib lrwxrwxrwx 1 jenkins jenkins 13 May 6 19:04 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K May 6 19:04 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 May 6 19:04 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K May 6 19:04 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 May 6 19:04 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 May 6 19:04 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M May 6 19:04 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 272M May 6 19:04 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M May 6 19:04 minio -rwxr-xr-x 1 jenkins jenkins 12M May 6 18:59 oauth2-server -rwxr-xr-x 1 jenkins jenkins 37M May 6 19:04 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M May 6 19:04 pd-ctl -rwxr-xr-x 1 jenkins jenkins 36M May 6 19:04 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 32M May 6 19:04 pd-recover -rwxr-xr-x 1 jenkins jenkins 106M May 6 19:04 pd-server -rwxr-xr-x 1 jenkins jenkins 26M May 6 19:04 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 3.0M May 6 19:04 pd-ut -rwxr-xr-x 1 jenkins jenkins 32M May 6 19:04 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 6 19:04 share -rwxr-xr-x 1 jenkins jenkins 32M May 6 19:04 stores-dump -rwxr-xr-x 1 jenkins jenkins 192M May 6 19:04 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 208M May 6 19:04 tidb-server -rwxr-xr-x 1 jenkins jenkins 380M May 6 19:04 tiflash -rwxr-xr-x 1 jenkins jenkins 418M May 6 19:04 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.0M May 6 19:04 xprog [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 70.84 secs (52659776 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-mnw-vn0pb [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55 [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9 [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-7w0-0lx49 [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55 is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-6cpdk-9l5jj --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2580ae2caf8361110d3a745b52747788c418ce2c" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-6cpdk" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@de9796b; decorates RemoteLauncher[hudson.remoting.Channel@24c905b7:JNLP4-connect connection from 10.233.88.222/10.233.88.222:41446] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-mnw-vn0pb is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-mnw7g-448r4 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "12e06cde63b90219092c103606c82b551d4f1728" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-mnw7g" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-mnw-vn0pb" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-mnw-vn0pb" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-mnw-vn0pb" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-mnw-vn0pb in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3 [Pipeline] node Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-7w0-0lx49 is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-7w05q-sr6nh --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "95f7ac5aa5fa905529a7689249cf80d11d07a8b2" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-7w05q" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-7w0-0lx49" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-7w0-0lx49" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-7w0-0lx49" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-7w0-0lx49 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@100a9517; decorates RemoteLauncher[hudson.remoting.Channel@2b86ac67:JNLP4-connect connection from 10.233.127.220/10.233.127.220:55606] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-c8j-cs84c [Pipeline] node Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9 is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-5lng6-75kfh --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "f89ac14cc68ed4c225e88401b52dbaacd4876946" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-5lng6" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@228450ca; decorates RemoteLauncher[hudson.remoting.Channel@4fe6c6c9:JNLP4-connect connection from 10.233.67.38/10.233.67.38:59874] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-j5h-9wmdc No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4c88e398; decorates RemoteLauncher[hudson.remoting.Channel@36101336:JNLP4-connect connection from 10.233.106.23/10.233.106.23:38642] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-s6fvx-8cx2r --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "b94141c8abce8cec65695d22ea6ae97a94b068d4" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-s6fvx" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-xp8-cvcxs [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { The recommended git tool is: git [Pipeline] cache > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-pfn35-99b1z --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7951da36a391c050d349496fd6749d21382c9129" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-pfn35" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-bpttb-sz7q9 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "0ec268d2b5fa69557b2cbe20fc4ef75065f942dc" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-bpttb" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@37fe5614; decorates RemoteLauncher[hudson.remoting.Channel@5d9f9621:JNLP4-connect connection from 10.233.127.170/10.233.127.170:53088] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-2p4-slmd8 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3 is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-c73tj-4czhc --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "22cb3c8bf9011fd664a265fcd05915ba6f11232a" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-c73tj" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Still waiting to schedule task โ€˜pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5โ€™ is offline Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-c8j-cs84c is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-c8jq9-mdmb4 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8e663edf024cd79238e33a96b221dfd60d2884e5" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-c8jq9" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-c8j-cs84c" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-c8j-cs84c" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-c8j-cs84c" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-c8j-cs84c in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Still waiting to schedule task Waiting for next available executor on โ€˜pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8mโ€™ Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-sb43l-m2hdh --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7ab45435f35abdc06fdcf187685e6a54eb952801" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-sb43l" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-xp8-cvcxs is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-xp897-80wql --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8de6d3cb46afe39575e3f9d8e22fc55209b11c66" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-xp897" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-xp8-cvcxs" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-xp8-cvcxs" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-xp8-cvcxs" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-xp8-cvcxs in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-j5h-9wmdc is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-j5hkm-wmp3g --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "4e05f98088c6824a47a842ff0d32f1a599238310" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-j5hkm" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-j5h-9wmdc" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-j5h-9wmdc" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-j5h-9wmdc" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-j5h-9wmdc in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Still waiting to schedule task โ€˜pingcap-tiflow-pull-cdc-integration-storage-test-1816-2p4-slmd8โ€™ is offline Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5 is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-404l8-vmdhz --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "4bad490619d8f65b1361793e4f22089d7b337316" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-404l8" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-2p4-slmd8 is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-2p408-9kwx5 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7f56d40a917b29fcc3d0023e5274b30f78a5d96b" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-2p408" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-2p4-slmd8" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-2p4-slmd8" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-2p4-slmd8" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-2p4-slmd8 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 7.39 secs (504549863 bytes/sec) [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] { [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G01 Run cases: http_api http_api_tls api_v2 http_api_tls_with_user_auth cli_tls_with_auth kafka_simple_basic kafka_simple_basic_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_simple_claim_check kafka_simple_claim_check_avro canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible multi_topics avro_basic canal_json_handle_key_only open_protocol_handle_key_only canal_json_claim_check open_protocol_claim_check canal_json_storage_basic canal_json_storage_partition_table multi_tables_ddl PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=c66ab943-7be8-4002-a6e2-3741f9506450 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-6cpdk GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55 pingcap_tiflow_pull_cdc_integration_storage_test_1816-6cpdk GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:06:02 CST 2024] <<<<<< run test case http_api success! >>>>>> [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] { [Pipeline] { The recommended git tool is: git [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-722-gnh4g Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-4h8-1vqcr [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-gxw-8pkgb [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@22c85c30; decorates RemoteLauncher[hudson.remoting.Channel@1744cbdb:JNLP4-connect connection from 10.233.88.150/10.233.88.150:34808] will be ignored (a typical symptom is the Git executable not being run inside a designated container) [Pipeline] dir Cloning the remote Git repository Using shallow clone with depth 1 Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@a2cab3e; decorates RemoteLauncher[hudson.remoting.Channel@6bf55df8:JNLP4-connect connection from 10.233.106.179/10.233.106.179:48744] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@60add708; decorates RemoteLauncher[hudson.remoting.Channel@4fc56d81:JNLP4-connect connection from 10.233.107.140/10.233.107.140:52706] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@42f63851; decorates RemoteLauncher[hudson.remoting.Channel@54881626:JNLP4-connect connection from 10.233.86.245/10.233.86.245:49384] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6d65c9f1; decorates RemoteLauncher[hudson.remoting.Channel@5d03bf1c:JNLP4-connect connection from 10.233.71.220/10.233.71.220:37778] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4aab7c2; decorates RemoteLauncher[hudson.remoting.Channel@532d70ad:JNLP4-connect connection from 10.233.105.67/10.233.105.67:55490] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@714c8617; decorates RemoteLauncher[hudson.remoting.Channel@2c2b9c48:JNLP4-connect connection from 10.233.68.17/10.233.68.17:56658] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@28c544af; decorates RemoteLauncher[hudson.remoting.Channel@5bad9924:JNLP4-connect connection from 10.233.105.192/10.233.105.192:53540] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3d511e8f; decorates RemoteLauncher[hudson.remoting.Channel@32a3b302:JNLP4-connect connection from 10.233.126.64/10.233.126.64:60922] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/http_api_tls/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:06:10 CST 2024] <<<<<< run test case http_api_tls success! >>>>>> Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35j Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 8.47 secs (440359472 bytes/sec) [Pipeline] { [Pipeline] cache Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-722-gnh4g is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-722k0-453xl --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "88d021fd6fa2d3d7f97e951410789f431c338de8" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-722k0" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-722-gnh4g" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-722-gnh4g" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-722-gnh4g" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-722-gnh4g in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/api_v2/run.sh using Sink-Type: storage... <<================= Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-4h8-1vqcr is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-4h8pq-c7znn --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "b4a4046bf9d11f02e060a3a9dde49e323fe71097" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-4h8pq" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-4h8-1vqcr" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-4h8-1vqcr" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-4h8-1vqcr" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-4h8-1vqcr in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Still waiting to schedule task Waiting for next available executor on โ€˜pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35jโ€™ Still waiting to schedule task โ€˜pingcap-tiflow-pull-cdc-integration-storage-test-1816-gxw-8pkgbโ€™ is offline Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35j is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-6jw4w-dhz0l --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "e407f9ede2917a1c8c4ce8cdb211fcc38f09f381" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-6jw4w" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35j" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35j" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35j" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35j in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/http_api_tls_with_user_auth/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:06:18 CST 2024] <<<<<< run test case http_api_tls_with_user_auth success! >>>>>> Agent pingcap-tiflow-pull-cdc-integration-storage-test-1816-gxw-8pkgb is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1816-gxwrn-0l2n5 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "42c6b7a1d0968f5b2997a842199d1f49e593c92c" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1816-gxwrn" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-gxw-8pkgb" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-gxw-8pkgb" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1816-gxw-8pkgb" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1816-gxw-8pkgb in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/cli_tls_with_auth/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 9.98 secs (373760336 bytes/sec) [Pipeline] { [Pipeline] cache start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365675540013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55, pid:1663, start at 2024-05-06 19:06:41.902197155 +0800 CST m=+5.337476962 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:08:41.908 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:06:41.877 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:56:41.877 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365675540013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55, pid:1663, start at 2024-05-06 19:06:41.902197155 +0800 CST m=+5.337476962 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:08:41.908 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:06:41.877 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:56:41.877 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365676c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55, pid:1738, start at 2024-05-06 19:06:41.997682107 +0800 CST m=+5.362703842 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:08:42.004 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:06:41.968 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:56:41.968 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting TLS PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 19.54 secs (190870276 bytes/sec) [Pipeline] { [Pipeline] cache Starting TLS TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting TLS TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365760ac0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55, pid:3339, start at 2024-05-06 19:06:56.943902297 +0800 CST m=+5.154055307 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:08:56.951 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:06:56.939 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:56:56.939 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + pd_host=127.0.0.1 + pd_port=2579 + is_tls=true + '[' true == true ']' ++ run_cdc_cli tso query --pd=https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3404.out cli tso query --pd=https://127.0.0.1:2579 + set +x + tso='449575287072751617 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575287072751617 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 8.43 secs (442313982 bytes/sec) [Pipeline] { [Pipeline] sh [Mon May 6 19:06:59 CST 2024] <<<<<< START cdc server in cli_tls_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates ']' + GO_FAILPOINTS= + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.34513453.out server --log-file /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_cli_tls_with_auth_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_data_cli_tls_with_auth_tls1 --cluster-id default --config /tmp/tidb_cdc_test/cli_tls_with_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G00 Run cases: bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_suicide server_config_compatibility changefeed_dup_error_restart kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro debezium lossy_ddl storage_csv_update PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=53e77dfd-0346-437f-bbe3-a75be80711aa BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-mnw-vn0pb HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-mnw7g GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1816-mnw7g pingcap-tiflow-pull-cdc-integration-storage-test-1816-mnw-vn0pb GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-mnw-vn0pb GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:00 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G05 Run cases: charset_gbk ddl_manager multi_source PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=1b37a85f-0d21-4c6a-8100-1e27ec3a8c69 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-7w0-0lx49 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-7w05q GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1816-7w05q pingcap-tiflow-pull-cdc-integration-storage-test-1816-7w0-0lx49 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-7w0-0lx49 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=b4f04fb6-72e1-4fd1-9cb7-ce5f2e22d3c2 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-5lng6 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1816-5lng6 pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:01 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G09 Run cases: gc_safepoint changefeed_pause_resume cli_with_auth savepoint synced_status PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=9719d679-9e9f-4e61-a294-1b2581135fc5 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-s6fvx GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1816-s6fvx pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:07:03 GMT < Content-Length: 1241 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73a873d2-2bce-47ad-88c0-0b2f3f2d7ac2 {"id":"73a873d2-2bce-47ad-88c0-0b2f3f2d7ac2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993620} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4d95b14083 73a873d2-2bce-47ad-88c0-0b2f3f2d7ac2 /tidb/cdc/default/default/upstream/7365841461608378836 {"id":7365841461608378836,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73a873d2-2bce-47ad-88c0-0b2f3f2d7ac2 {"id":"73a873d2-2bce-47ad-88c0-0b2f3f2d7ac2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993620} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4d95b14083 73a873d2-2bce-47ad-88c0-0b2f3f2d7ac2 /tidb/cdc/default/default/upstream/7365841461608378836 {"id":7365841461608378836,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73a873d2-2bce-47ad-88c0-0b2f3f2d7ac2 {"id":"73a873d2-2bce-47ad-88c0-0b2f3f2d7ac2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993620} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4d95b14083 73a873d2-2bce-47ad-88c0-0b2f3f2d7ac2 /tidb/cdc/default/default/upstream/7365841461608378836 {"id":7365841461608378836,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@534c236b; decorates RemoteLauncher[hudson.remoting.Channel@67e50e3:JNLP4-connect connection from 10.233.84.61/10.233.84.61:34068] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:03 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@49ecf868; decorates RemoteLauncher[hudson.remoting.Channel@3aa9ff95:JNLP4-connect connection from 10.233.67.118/10.233.67.118:34968] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4d8c12b; decorates RemoteLauncher[hudson.remoting.Channel@637f439d:JNLP4-connect connection from 10.233.72.11/10.233.72.11:40188] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... [Pipeline] { + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3506.out cli changefeed create --start-ts=449575287072751617 '--sink-uri=file:///tmp/tidb_cdc_test/cli_tls_with_auth/storage_test/ticdc-cli-test-22694?protocol=canal-json&enable-tidb-extension=true' --tz=Asia/Shanghai -c=custom-changefeed-name Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] cache Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [WARN] --tz is deprecated in changefeed settings. start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@746cffd1; decorates RemoteLauncher[hudson.remoting.Channel@2161d80d:JNLP4-connect connection from 10.233.100.5/10.233.100.5:51816] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365841461608378836,"namespace":"default","id":"custom-changefeed-name","sink_uri":"file:///tmp/tidb_cdc_test/cli_tls_with_auth/storage_test/ticdc-cli-test-22694?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:07:05.784093053+08:00","start_ts":449575287072751617,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575287072751617,"checkpoint_ts":449575287072751617,"checkpoint_time":"2024-05-06 19:06:58.289"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: storage... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: storage... <<================= kafka downstream isn't support syncpoint record [Mon May 6 19:07:06 CST 2024] <<<<<< run test case syncpoint success! >>>>>> Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide + set +x + workdir=/tmp/tidb_cdc_test/cli_tls_with_auth + sink_uri='file:///tmp/tidb_cdc_test/cli_tls_with_auth/storage_test/ticdc-cli-test-22694?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/cli_tls_with_auth ++ date + echo '[Mon May 6 19:07:07 CST 2024] <<<<<< START storage consumer in cli_tls_with_auth case >>>>>>' [Mon May 6 19:07:07 CST 2024] <<<<<< START storage consumer in cli_tls_with_auth case >>>>>> + cd /tmp/tidb_cdc_test/cli_tls_with_auth + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/cli_tls_with_auth + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/cli_tls_with_auth/storage_test/ticdc-cli-test-22694?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table test.simple not exists for 1-th check, retry later > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCE19E4E2E7FCF < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 11:07:08 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Mon May 6 19:07:08 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.simple not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:10 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> Commit message: "fix(br): use failpoint tidb-server instead (#2951)" ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.simple not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:13 CST 2024] <<<<<< run test case server_config_compatibility success! >>>>>> table test.simple not exists for 4-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:14 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.simple not exists for 5-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/changefeed_dup_error_restart/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:16 CST 2024] <<<<<< run test case changefeed_dup_error_restart success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365874700013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-7w0-0lx49, pid:1228, start at 2024-05-06 19:07:14.623723205 +0800 CST m=+5.524517981 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:09:14.632 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:07:14.588 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:57:14.588 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365874700013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-7w0-0lx49, pid:1228, start at 2024-05-06 19:07:14.623723205 +0800 CST m=+5.524517981 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:09:14.632 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:07:14.588 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:57:14.588 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365875e8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-7w0-0lx49, pid:1298, start at 2024-05-06 19:07:14.697603373 +0800 CST m=+5.521846327 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:09:14.704 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:07:14.682 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:57:14.682 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365890e00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:1169, start at 2024-05-06 19:07:16.412327337 +0800 CST m=+5.257229952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:09:16.419 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:07:16.408 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:57:16.408 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.simple not exists for 6-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:18 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.simple exists table test.`simple-dash` exists + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365841461608378836, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575292472918018, "checkpoint_time": "2024-05-06 19:07:18.889", "error": null }' + echo '{ "upstream_id": 7365841461608378836, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575292472918018, "checkpoint_time": "2024-05-06 19:07:18.889", "error": null }' { "upstream_id": 7365841461608378836, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575292472918018, "checkpoint_time": "2024-05-06 19:07:18.889", "error": null } ++ echo '{' '"upstream_id":' 7365841461608378836, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575292472918018, '"checkpoint_time":' '"2024-05-06' '19:07:18.889",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365841461608378836, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575292472918018, '"checkpoint_time":' '"2024-05-06' '19:07:18.889",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365890e00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:1169, start at 2024-05-06 19:07:16.412327337 +0800 CST m=+5.257229952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:09:16.419 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:07:16.408 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:57:16.408 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365890c80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:1243, start at 2024-05-06 19:07:16.438184218 +0800 CST m=+5.203987841 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:09:16.447 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:07:16.452 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:57:16.452 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } changefeed count 1 check pass, pd_addr: https://127.0.0.1:2579 [Mon May 6 19:07:19 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.26282630.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:19 CST 2024] <<<<<< run test case kafka_big_messages success! >>>>>> Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc001417b60} {Type:update Path:[Config SyncPointRetention] From: To:0xc001417b68} {Type:update Path:[Config Consistent] From: To:0xc00127e460} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3798.out cli changefeed --changefeed-id custom-changefeed-name pause PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [Mon May 6 19:07:21 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.25862588.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:07:22 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e34cd4e-f02b-49a9-84a2-5f12736e5830 {"id":"4e34cd4e-f02b-49a9-84a2-5f12736e5830","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993639} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d95f5a4cc 4e34cd4e-f02b-49a9-84a2-5f12736e5830 /tidb/cdc/default/default/upstream/7365841535595516942 {"id":7365841535595516942,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e34cd4e-f02b-49a9-84a2-5f12736e5830 {"id":"4e34cd4e-f02b-49a9-84a2-5f12736e5830","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993639} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d95f5a4cc 4e34cd4e-f02b-49a9-84a2-5f12736e5830 /tidb/cdc/default/default/upstream/7365841535595516942 {"id":7365841535595516942,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e34cd4e-f02b-49a9-84a2-5f12736e5830 {"id":"4e34cd4e-f02b-49a9-84a2-5f12736e5830","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993639} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d95f5a4cc 4e34cd4e-f02b-49a9-84a2-5f12736e5830 /tidb/cdc/default/default/upstream/7365841535595516942 {"id":7365841535595516942,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: d55a6fde-12f1-48b5-8c96-8626b8592543 Info: {"upstream_id":7365841535595516942,"namespace":"default","id":"d55a6fde-12f1-48b5-8c96-8626b8592543","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:07:22.842181867+08:00","start_ts":449575292630204417,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575292630204417,"checkpoint_ts":449575292630204417,"checkpoint_time":"2024-05-06 19:07:19.489"} + workdir=/tmp/tidb_cdc_test/charset_gbk + sink_uri=mysql://normal:123456@127.0.0.1:3306/ + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/charset_gbk ++ date + echo '[Mon May 6 19:07:22 CST 2024] <<<<<< START storage consumer in charset_gbk case >>>>>>' [Mon May 6 19:07:22 CST 2024] <<<<<< START storage consumer in charset_gbk case >>>>>> + cd /tmp/tidb_cdc_test/charset_gbk + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/charset_gbk + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/charset_gbk/cdc_storage_consumer.log --log-level debug --upstream-uri mysql://normal:123456@127.0.0.1:3306/ --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:23 CST 2024] <<<<<< run test case kafka_compression success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:07:24 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f8ca6d4-0762-4bad-89f7-8e1817261709 {"id":"4f8ca6d4-0762-4bad-89f7-8e1817261709","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993641} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d95f7decb 4f8ca6d4-0762-4bad-89f7-8e1817261709 /tidb/cdc/default/default/upstream/7365841545495017976 {"id":7365841545495017976,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f8ca6d4-0762-4bad-89f7-8e1817261709 {"id":"4f8ca6d4-0762-4bad-89f7-8e1817261709","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993641} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d95f7decb 4f8ca6d4-0762-4bad-89f7-8e1817261709 /tidb/cdc/default/default/upstream/7365841545495017976 {"id":7365841545495017976,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f8ca6d4-0762-4bad-89f7-8e1817261709 {"id":"4f8ca6d4-0762-4bad-89f7-8e1817261709","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993641} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d95f7decb 4f8ca6d4-0762-4bad-89f7-8e1817261709 /tidb/cdc/default/default/upstream/7365841545495017976 {"id":7365841545495017976,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/gc_safepoint + sink_uri='file:///tmp/tidb_cdc_test/gc_safepoint/storage_test/ticdc-gc-safepoint-5061?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/gc_safepoint ++ date + echo '[Mon May 6 19:07:24 CST 2024] <<<<<< START storage consumer in gc_safepoint case >>>>>>' [Mon May 6 19:07:24 CST 2024] <<<<<< START storage consumer in gc_safepoint case >>>>>> + cd /tmp/tidb_cdc_test/gc_safepoint + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/gc_safepoint + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/gc_safepoint/storage_test/ticdc-gc-safepoint-5061?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' 0 check diff failed 1-th time, retry later + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365841461608378836, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449575292997206017, "checkpoint_time": "2024-05-06 19:07:20.889", "error": null }' + echo '{ "upstream_id": 7365841461608378836, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449575292997206017, "checkpoint_time": "2024-05-06 19:07:20.889", "error": null }' { "upstream_id": 7365841461608378836, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449575292997206017, "checkpoint_time": "2024-05-06 19:07:20.889", "error": null } ++ echo '{' '"upstream_id":' 7365841461608378836, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449575292997206017, '"checkpoint_time":' '"2024-05-06' '19:07:20.889",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365841461608378836, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449575292997206017, '"checkpoint_time":' '"2024-05-06' '19:07:20.889",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3893.out cli changefeed update --pd=https://127.0.0.1:2579 --config=/tmp/tidb_cdc_test/cli_tls_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:25 CST 2024] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:26 CST 2024] <<<<<< run test case kafka_messages success! >>>>>> Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc00196cb68} {Type:update Path:[Config SyncPointRetention] From: To:0xc00196cb78} {Type:update Path:[Config Consistent] From: To:0xc0013a1960} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7365841461608378836,"namespace":"default","id":"custom-changefeed-name","sink_uri":"file:///tmp/tidb_cdc_test/cli_tls_with_auth/storage_test/ticdc-cli-test-22694?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:07:05.784093053+08:00","start_ts":449575287072751617,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":0,"checkpoint_ts":449575292997206017,"checkpoint_time":"2024-05-06 19:07:20.889"} PASS coverage: 2.8% of statements in github.com/pingcap/tiflow/... check diff failed 2-th time, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3933.out cli changefeed --changefeed-id custom-changefeed-name resume Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 17.31 secs (215519429 bytes/sec) [Pipeline] { [Pipeline] cache check diff failed 3-th time, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: storage... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:29 CST 2024] <<<<<< run test case kafka_sink_error_resume success! >>>>>> You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide + set +x check diff failed 4-th time, retry later * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCE1A3C0882E76 < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 11:07:31 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Mon May 6 19:07:31 CST 2024] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Exiting on signal: INTERRUPT find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:32 CST 2024] <<<<<< run test case mq_sink_lost_callback success! >>>>>> check diff failed 5-th time, retry later + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365841461608378836, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575296142934018, "checkpoint_time": "2024-05-06 19:07:32.889", "error": null }' + echo '{ "upstream_id": 7365841461608378836, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575296142934018, "checkpoint_time": "2024-05-06 19:07:32.889", "error": null }' { "upstream_id": 7365841461608378836, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575296142934018, "checkpoint_time": "2024-05-06 19:07:32.889", "error": null } ++ echo '{' '"upstream_id":' 7365841461608378836, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575296142934018, '"checkpoint_time":' '"2024-05-06' '19:07:32.889",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365841461608378836, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575296142934018, '"checkpoint_time":' '"2024-05-06' '19:07:32.889",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4018.out cli changefeed --changefeed-id custom-changefeed-name remove Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 449575296405078019 SinkURI: file:///tmp/tidb_cdc_test/cli_tls_with_auth/storage_test/ticdc-cli-test-22694?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff successfully check_safepoint_forward http://127.0.0.1:2379 7365841545495017976 449575296488964097 449575293985488898 + set +x find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:36 CST 2024] <<<<<< run test case mq_sink_dispatcher success! >>>>>> run task successfully check_changefeed_state http://127.0.0.1:2379 c572aa66-b488-4137-9092-b5219a392364 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=c572aa66-b488-4137-9092-b5219a392364 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c c572aa66-b488-4137-9092-b5219a392364 -s =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:36 CST 2024] <<<<<< run test case consistent_partition_table success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7365841545495017976, "namespace": "default", "id": "c572aa66-b488-4137-9092-b5219a392364", "state": "stopped", "checkpoint_tso": 449575297012989955, "checkpoint_time": "2024-05-06 19:07:36.208", "error": null }' + echo '{ "upstream_id": 7365841545495017976, "namespace": "default", "id": "c572aa66-b488-4137-9092-b5219a392364", "state": "stopped", "checkpoint_tso": 449575297012989955, "checkpoint_time": "2024-05-06 19:07:36.208", "error": null }' { "upstream_id": 7365841545495017976, "namespace": "default", "id": "c572aa66-b488-4137-9092-b5219a392364", "state": "stopped", "checkpoint_tso": 449575297012989955, "checkpoint_time": "2024-05-06 19:07:36.208", "error": null } ++ echo '{' '"upstream_id":' 7365841545495017976, '"namespace":' '"default",' '"id":' '"c572aa66-b488-4137-9092-b5219a392364",' '"state":' '"stopped",' '"checkpoint_tso":' 449575297012989955, '"checkpoint_time":' '"2024-05-06' '19:07:36.208",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365841545495017976, '"namespace":' '"default",' '"id":' '"c572aa66-b488-4137-9092-b5219a392364",' '"state":' '"stopped",' '"checkpoint_tso":' 449575297012989955, '"checkpoint_time":' '"2024-05-06' '19:07:36.208",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365841545495017976 table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later changefeed count 0 check pass, pd_addr: https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4095.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/cli_tls_with_auth/storage_test/ticdc-cli-test-22694?protocol=canal-json&enable-tidb-extension=true' --tz=Asia/Shanghai -c=custom-changefeed-name find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:39 CST 2024] <<<<<< run test case kafka_column_selector success! >>>>>> run task successfully [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365841461608378836,"namespace":"default","id":"custom-changefeed-name","sink_uri":"file:///tmp/tidb_cdc_test/cli_tls_with_auth/storage_test/ticdc-cli-test-22694?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:07:40.134034618+08:00","start_ts":449575297991049223,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575297991049223,"checkpoint_ts":449575297991049223,"checkpoint_time":"2024-05-06 19:07:39.939"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check_changefeed_state http://127.0.0.1:2379 c572aa66-b488-4137-9092-b5219a392364 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=c572aa66-b488-4137-9092-b5219a392364 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c c572aa66-b488-4137-9092-b5219a392364 -s table test.finish_mark not exists for 3-th check, retry later + info='{ "upstream_id": 7365841545495017976, "namespace": "default", "id": "c572aa66-b488-4137-9092-b5219a392364", "state": "normal", "checkpoint_tso": 449575297012989955, "checkpoint_time": "2024-05-06 19:07:36.208", "error": null }' + echo '{ "upstream_id": 7365841545495017976, "namespace": "default", "id": "c572aa66-b488-4137-9092-b5219a392364", "state": "normal", "checkpoint_tso": 449575297012989955, "checkpoint_time": "2024-05-06 19:07:36.208", "error": null }' { "upstream_id": 7365841545495017976, "namespace": "default", "id": "c572aa66-b488-4137-9092-b5219a392364", "state": "normal", "checkpoint_tso": 449575297012989955, "checkpoint_time": "2024-05-06 19:07:36.208", "error": null } ++ echo '{' '"upstream_id":' 7365841545495017976, '"namespace":' '"default",' '"id":' '"c572aa66-b488-4137-9092-b5219a392364",' '"state":' '"normal",' '"checkpoint_tso":' 449575297012989955, '"checkpoint_time":' '"2024-05-06' '19:07:36.208",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365841545495017976, '"namespace":' '"default",' '"id":' '"c572aa66-b488-4137-9092-b5219a392364",' '"state":' '"normal",' '"checkpoint_tso":' 449575297012989955, '"checkpoint_time":' '"2024-05-06' '19:07:36.208",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7365841545495017976 449575297012989954 449575297012989955 + set +x run task successfully check_changefeed_state http://127.0.0.1:2379 c572aa66-b488-4137-9092-b5219a392364 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=c572aa66-b488-4137-9092-b5219a392364 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c c572aa66-b488-4137-9092-b5219a392364 -s + info='{ "upstream_id": 7365841545495017976, "namespace": "default", "id": "c572aa66-b488-4137-9092-b5219a392364", "state": "stopped", "checkpoint_tso": 449575297012989955, "checkpoint_time": "2024-05-06 19:07:36.208", "error": null }' + echo '{ "upstream_id": 7365841545495017976, "namespace": "default", "id": "c572aa66-b488-4137-9092-b5219a392364", "state": "stopped", "checkpoint_tso": 449575297012989955, "checkpoint_time": "2024-05-06 19:07:36.208", "error": null }' { "upstream_id": 7365841545495017976, "namespace": "default", "id": "c572aa66-b488-4137-9092-b5219a392364", "state": "stopped", "checkpoint_tso": 449575297012989955, "checkpoint_time": "2024-05-06 19:07:36.208", "error": null } ++ echo '{' '"upstream_id":' 7365841545495017976, '"namespace":' '"default",' '"id":' '"c572aa66-b488-4137-9092-b5219a392364",' '"state":' '"stopped",' '"checkpoint_tso":' 449575297012989955, '"checkpoint_time":' '"2024-05-06' '19:07:36.208",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365841545495017976, '"namespace":' '"default",' '"id":' '"c572aa66-b488-4137-9092-b5219a392364",' '"state":' '"stopped",' '"checkpoint_tso":' 449575297012989955, '"checkpoint_time":' '"2024-05-06' '19:07:36.208",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state http://127.0.0.1:2379 1fb98d2b-f776-48c5-a67c-c6bc5ee52104 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=1fb98d2b-f776-48c5-a67c-c6bc5ee52104 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 1fb98d2b-f776-48c5-a67c-c6bc5ee52104 -s table test.finish_mark exists check table exists success check diff successfully + info='{ "upstream_id": 7365841545495017976, "namespace": "default", "id": "1fb98d2b-f776-48c5-a67c-c6bc5ee52104", "state": "normal", "checkpoint_tso": 449575298573008902, "checkpoint_time": "2024-05-06 19:07:42.159", "error": null }' + echo '{ "upstream_id": 7365841545495017976, "namespace": "default", "id": "1fb98d2b-f776-48c5-a67c-c6bc5ee52104", "state": "normal", "checkpoint_tso": 449575298573008902, "checkpoint_time": "2024-05-06 19:07:42.159", "error": null }' { "upstream_id": 7365841545495017976, "namespace": "default", "id": "1fb98d2b-f776-48c5-a67c-c6bc5ee52104", "state": "normal", "checkpoint_tso": 449575298573008902, "checkpoint_time": "2024-05-06 19:07:42.159", "error": null } ++ echo '{' '"upstream_id":' 7365841545495017976, '"namespace":' '"default",' '"id":' '"1fb98d2b-f776-48c5-a67c-c6bc5ee52104",' '"state":' '"normal",' '"checkpoint_tso":' 449575298573008902, '"checkpoint_time":' '"2024-05-06' '19:07:42.159",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365841545495017976, '"namespace":' '"default",' '"id":' '"1fb98d2b-f776-48c5-a67c-c6bc5ee52104",' '"state":' '"normal",' '"checkpoint_tso":' 449575298573008902, '"checkpoint_time":' '"2024-05-06' '19:07:42.159",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365841545495017976 wait process cdc.test exit for 1-th time... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_column_selector_avro/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:42 CST 2024] <<<<<< run test case kafka_column_selector_avro success! >>>>>> wait process cdc.test exit for 2-th time... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_big_messages_v2/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:42 CST 2024] <<<<<< run test case kafka_big_messages_v2 success! >>>>>> wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:07:44 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365841461608378836, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575299026518019, "checkpoint_time": "2024-05-06 19:07:43.889", "error": null }' + echo '{ "upstream_id": 7365841461608378836, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575299026518019, "checkpoint_time": "2024-05-06 19:07:43.889", "error": null }' { "upstream_id": 7365841461608378836, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575299026518019, "checkpoint_time": "2024-05-06 19:07:43.889", "error": null } ++ echo '{' '"upstream_id":' 7365841461608378836, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575299026518019, '"checkpoint_time":' '"2024-05-06' '19:07:43.889",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365841461608378836, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575299026518019, '"checkpoint_time":' '"2024-05-06' '19:07:43.889",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/multi_tables_ddl_v2/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:45 CST 2024] <<<<<< run test case multi_tables_ddl_v2 success! >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4217.out cli unsafe delete-service-gc-safepoint Changefeed remove successfully. ID: c572aa66-b488-4137-9092-b5219a392364 CheckpointTs: 449575297012989955 SinkURI: file:///tmp/tidb_cdc_test/gc_safepoint/storage_test/ticdc-gc-safepoint-5061?protocol=canal-json&enable-tidb-extension=true check_safepoint_forward http://127.0.0.1:2379 7365841545495017976 449575298573008901 449575298573008902 449575297012989955 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/debezium/run.sh using Sink-Type: storage... <<================= find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 15.47 secs (241210977 bytes/sec) [Pipeline] { [Pipeline] cache run task successfully Changefeed remove successfully. ID: 1fb98d2b-f776-48c5-a67c-c6bc5ee52104 CheckpointTs: 449575299634692101 SinkURI: file:///tmp/tidb_cdc_test/gc_safepoint/storage_test/ticdc-gc-safepoint-5061?protocol=canal-json&enable-tidb-extension=true check_safepoint_cleared http://127.0.0.1:2379 7365841545495017976 run task successfully + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4255.out cli unsafe reset --no-confirm --pd=https://127.0.0.1:2579 wait process cdc.test exit for 1-th time... reset and all metadata truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:07:49 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/lossy_ddl Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4375.out cli unsafe resolve-lock --region=42 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:07:53 CST 2024] <<<<<< run test case multi_topics_v2 success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4413.out cli unsafe resolve-lock --region=42 --ts=449575300652072964 Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 1585 44 --:--:-- --:--:-- --:--:-- 1593 { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: storage... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup + CDC_BINARY=cdc.test + SINK_TYPE=storage + EXIST_FILES=() + CLEANED_FILES=() + trap stop_tidb_cluster EXIT + run storage + '[' storage '!=' storage ']' + rm -rf /tmp/tidb_cdc_test/storage_cleanup + mkdir -p /tmp/tidb_cdc_test/storage_cleanup + start_tidb_cluster --workdir /tmp/tidb_cdc_test/storage_cleanup The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 181 100 181 0 0 1108 0 --:--:-- --:--:-- --:--:-- 1110 { "version": "v8.2.0-alpha-25-g3c107706d", "git_hash": "3c107706d2612c5f422cde6480fd92425795842e", "id": "31050403-06e4-4cc2-a68e-da442a249a89", "pid": 3456, "is_owner": true start tidb cluster in /tmp/tidb_cdc_test/storage_cleanup Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release }wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365b42500016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-mnw-vn0pb, pid:2488, start at 2024-05-06 19:08:00.549067069 +0800 CST m=+5.172963056 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:00.557 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:00.532 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:00.532 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365b42500016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-mnw-vn0pb, pid:2488, start at 2024-05-06 19:08:00.549067069 +0800 CST m=+5.172963056 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:00.557 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:00.532 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:00.532 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365b455c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-mnw-vn0pb, pid:2568, start at 2024-05-06 19:08:00.728073273 +0800 CST m=+5.280850627 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:00.733 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:00.727 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:00.727 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/lossy_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/lossy_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/lossy_ddl/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/lossy_ddl/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/lossy_ddl/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:08:02 CST 2024] <<<<<< run test case cli_tls_with_auth success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 19:08:03 CST 2024] <<<<<< START cdc server in lossy_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.lossy_ddl.37873789.out server --log-file /tmp/tidb_cdc_test/lossy_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/lossy_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 17.37 secs (214822256 bytes/sec) [Pipeline] { [Pipeline] cache Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:08:06 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb6b7d1f-740e-43b5-ab4b-c5d3e0dd9aaa {"id":"eb6b7d1f-740e-43b5-ab4b-c5d3e0dd9aaa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993684} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96a49ec5 eb6b7d1f-740e-43b5-ab4b-c5d3e0dd9aaa /tidb/cdc/default/default/upstream/7365841737004073559 {"id":7365841737004073559,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb6b7d1f-740e-43b5-ab4b-c5d3e0dd9aaa {"id":"eb6b7d1f-740e-43b5-ab4b-c5d3e0dd9aaa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993684} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96a49ec5 eb6b7d1f-740e-43b5-ab4b-c5d3e0dd9aaa /tidb/cdc/default/default/upstream/7365841737004073559 {"id":7365841737004073559,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb6b7d1f-740e-43b5-ab4b-c5d3e0dd9aaa {"id":"eb6b7d1f-740e-43b5-ab4b-c5d3e0dd9aaa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993684} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96a49ec5 eb6b7d1f-740e-43b5-ab4b-c5d3e0dd9aaa /tidb/cdc/default/default/upstream/7365841737004073559 {"id":7365841737004073559,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 489e702a-63c2-40bc-b759-07ead2f215bb Info: {"upstream_id":7365841737004073559,"namespace":"default","id":"489e702a-63c2-40bc-b759-07ead2f215bb","sink_uri":"blackhole:","create_time":"2024-05-06T19:08:06.942274022+08:00","start_ts":449575305041149957,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575305041149957,"checkpoint_ts":449575305041149957,"checkpoint_time":"2024-05-06 19:08:06.833"} ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365bbdf80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-7w0-0lx49, pid:3733, start at 2024-05-06 19:08:08.464654657 +0800 CST m=+5.644536800 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:08.474 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:08.446 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:08.446 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365bbdf80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-7w0-0lx49, pid:3733, start at 2024-05-06 19:08:08.464654657 +0800 CST m=+5.644536800 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:08.474 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:08.446 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:08.446 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365bbf600005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-7w0-0lx49, pid:3810, start at 2024-05-06 19:08:08.539836355 +0800 CST m=+5.640093403 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:08.547 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:08.536 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:08.536 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 19:08:13 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.51765178.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_simple_basic/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:08:14 CST 2024] <<<<<< run test case kafka_simple_basic success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365bf5e00008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9, pid:2301, start at 2024-05-06 19:08:12.032034836 +0800 CST m=+5.565774056 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:12.039 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:12.024 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:12.024 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365bf5e00008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9, pid:2301, start at 2024-05-06 19:08:12.032034836 +0800 CST m=+5.565774056 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:12.039 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:12.024 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:12.024 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365bf75c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9, pid:2376, start at 2024-05-06 19:08:12.133372076 +0800 CST m=+5.594046950 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:12.144 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:12.119 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:12.119 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/storage_cleanup/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/storage_cleanup/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/storage_cleanup/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/storage_cleanup/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/storage_cleanup/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:08:16 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c669a935-7135-4ef6-bf96-7646129b7145 {"id":"c669a935-7135-4ef6-bf96-7646129b7145","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993693} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96c570cc c669a935-7135-4ef6-bf96-7646129b7145 /tidb/cdc/default/default/upstream/7365841762574137728 {"id":7365841762574137728,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c669a935-7135-4ef6-bf96-7646129b7145 {"id":"c669a935-7135-4ef6-bf96-7646129b7145","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993693} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96c570cc c669a935-7135-4ef6-bf96-7646129b7145 /tidb/cdc/default/default/upstream/7365841762574137728 {"id":7365841762574137728,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c669a935-7135-4ef6-bf96-7646129b7145 {"id":"c669a935-7135-4ef6-bf96-7646129b7145","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993693} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96c570cc c669a935-7135-4ef6-bf96-7646129b7145 /tidb/cdc/default/default/upstream/7365841762574137728 {"id":7365841762574137728,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.5242.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-5438?protocol=canal-json&enable-tidb-extension=true' -c=ddl-manager + cd /tmp/tidb_cdc_test/storage_cleanup + run_cdc_server --workdir /tmp/tidb_cdc_test/storage_cleanup --binary cdc.test [Mon May 6 19:08:16 CST 2024] <<<<<< START cdc server in storage_cleanup case >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7365841762574137728,"namespace":"default","id":"ddl-manager","sink_uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-5438?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:08:16.764005643+08:00","start_ts":449575307613569027,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575307613569027,"checkpoint_ts":449575307613569027,"checkpoint_time":"2024-05-06 19:08:16.646"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 1-th time, retry later + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_cleanup.36223624.out server --log-file /tmp/tidb_cdc_test/storage_cleanup/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/storage_cleanup/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 2-th time, retry later + set +x + workdir=/tmp/tidb_cdc_test/ddl_manager + sink_uri='file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-5438?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/ddl_manager ++ date + echo '[Mon May 6 19:08:18 CST 2024] <<<<<< START storage consumer in ddl_manager case >>>>>>' [Mon May 6 19:08:18 CST 2024] <<<<<< START storage consumer in ddl_manager case >>>>>> + cd /tmp/tidb_cdc_test/ddl_manager + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/ddl_manager + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/ddl_manager/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-5438?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365c2e600006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:4526, start at 2024-05-06 19:08:15.646883054 +0800 CST m=+5.472812232 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:15.654 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:15.640 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:15.640 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365c2e600006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:4526, start at 2024-05-06 19:08:15.646883054 +0800 CST m=+5.472812232 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:15.654 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:15.640 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:15.640 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365c2d880014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:4598, start at 2024-05-06 19:08:15.626485421 +0800 CST m=+5.388692790 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:15.637 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:15.636 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:15.636 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:08:19 CST 2024] <<<<<< run test case kafka_simple_basic_avro success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:08:19 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/84b3eb4a-1000-4ae9-a2ce-4c51089071dc {"id":"84b3eb4a-1000-4ae9-a2ce-4c51089071dc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993697} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96cfefca 84b3eb4a-1000-4ae9-a2ce-4c51089071dc /tidb/cdc/default/default/upstream/7365841784839173240 {"id":7365841784839173240,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/84b3eb4a-1000-4ae9-a2ce-4c51089071dc {"id":"84b3eb4a-1000-4ae9-a2ce-4c51089071dc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993697} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96cfefca 84b3eb4a-1000-4ae9-a2ce-4c51089071dc /tidb/cdc/default/default/upstream/7365841784839173240 {"id":7365841784839173240,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/84b3eb4a-1000-4ae9-a2ce-4c51089071dc {"id":"84b3eb4a-1000-4ae9-a2ce-4c51089071dc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993697} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96cfefca 84b3eb4a-1000-4ae9-a2ce-4c51089071dc /tidb/cdc/default/default/upstream/7365841784839173240 {"id":7365841784839173240,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + generate_historic_files + local target_bucket=storage_test ++ date -d yesterday +%Y-%m-%d + yesterday=2024-05-05 ++ date -d '2 days ago' +%Y-%m-%d + day_before_yesterday=2024-05-04 + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test multi_data_type 2024-05-05 10 false + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test + local table=multi_data_type + local day=2024-05-05 + local file_cnt=10 + local should_clean=false + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/1.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/2.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/3.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/4.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/5.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/6.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/7.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/8.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/9.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/10.data + '[' false == true ']' + EXIST_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test multi_charset 2024-05-04 10 true + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test + local table=multi_charset + local day=2024-05-04 + local file_cnt=10 + local should_clean=true + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/1.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/2.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/3.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/4.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/5.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/6.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/7.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/8.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/9.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/10.data + '[' true == true ']' + CLEANED_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test binary_columns 2024-05-04 10 true + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test + local table=binary_columns + local day=2024-05-04 + local file_cnt=10 + local should_clean=true + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/1.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/2.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/3.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/4.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/5.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/6.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/7.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/8.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/9.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/10.data + '[' true == true ']' + CLEANED_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test multi_data_type_dummy 2024-05-04 10 true + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test + local table=multi_data_type_dummy + local day=2024-05-04 + local file_cnt=10 + local should_clean=true + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/1.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/2.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/3.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/4.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/5.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/6.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/7.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/8.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/9.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/10.data + '[' true == true ']' + CLEANED_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test multi_charset_dummy 2024-05-04 10 true + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test + local table=multi_charset_dummy + local day=2024-05-04 + local file_cnt=10 + local should_clean=true + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/1.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/2.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/3.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/4.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/5.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/6.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/7.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/8.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/9.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/10.data + '[' true == true ']' + CLEANED_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test binary_columns_dummy 2024-05-05 10 false + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test + local table=binary_columns_dummy + local day=2024-05-05 + local file_cnt=10 + local should_clean=false + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/1.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/2.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/3.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/4.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/5.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/6.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/7.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/8.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/9.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/10.data + '[' false == true ']' + EXIST_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test2 multi_data_type 2024-05-04 10 true + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test2 + local table=multi_data_type + local day=2024-05-04 + local file_cnt=10 + local should_clean=true + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/1.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/2.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/3.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/4.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/5.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/6.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/7.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/8.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/9.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/10.data + '[' true == true ']' + CLEANED_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test2 multi_charset 2024-05-04 10 true + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test2 + local table=multi_charset + local day=2024-05-04 + local file_cnt=10 + local should_clean=true + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/1.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/2.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/3.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/4.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/5.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/6.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/7.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/8.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/9.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/10.data + '[' true == true ']' + CLEANED_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test2 binary_columns 2024-05-05 10 false + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test2 + local table=binary_columns + local day=2024-05-05 + local file_cnt=10 + local should_clean=false + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/1.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/2.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/3.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/4.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/5.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/6.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/7.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/8.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/9.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/10.data + '[' false == true ']' + EXIST_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test_default test multi_data_type 2022-01-01 10 false + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test_default + local schema=test + local table=multi_data_type + local day=2022-01-01 + local file_cnt=10 + local should_clean=false + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/1.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/2.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/3.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/4.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/5.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/6.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/7.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/8.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/9.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/10.data + '[' false == true ']' + EXIST_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test_default test multi_charset 2022-01-02 10 false + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test_default + local schema=test + local table=multi_charset + local day=2022-01-02 + local file_cnt=10 + local should_clean=false + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/1.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/2.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/3.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/4.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/5.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/6.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/7.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/8.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/9.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/10.data + '[' false == true ']' + EXIST_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test_default test binary_columns 2022-01-03 10 false + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test_default + local schema=test + local table=binary_columns + local day=2022-01-03 + local file_cnt=10 + local should_clean=false + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/1.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/2.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/3.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/4.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/5.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/6.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/7.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/8.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/9.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/10.data + '[' false == true ']' + EXIST_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/meta/CDC.index + SINK_URI_DEFAULT='file:///tmp/tidb_cdc_test/storage_cleanup/storage_test_default?flush-interval=5s' + run_cdc_cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/storage_cleanup/storage_test_default?flush-interval=5s' -c default-config-test --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed-default.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_cleanup.cli.3848.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/storage_cleanup/storage_test_default?flush-interval=5s' -c default-config-test --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed-default.toml Create changefeed successfully! ID: default-config-test Info: {"upstream_id":7365841784839173240,"namespace":"default","id":"default-config-test","sink_uri":"file:///tmp/tidb_cdc_test/storage_cleanup/storage_test_default?flush-interval=5s","create_time":"2024-05-06T19:08:20.519581491+08:00","start_ts":449575308591104005,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575308591104005,"checkpoint_ts":449575308591104005,"checkpoint_time":"2024-05-06 19:08:20.375"} PASS [Mon May 6 19:08:20 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.59105912.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 coverage: 2.5% of statements in github.com/pingcap/tiflow/... check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 3-th time, retry later + set +x + sleep 20 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:08:22 CST 2024] <<<<<< run test case kafka_simple_handle_key_only success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:08:23 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28cb6861-304f-4323-bb9e-ef4e95aed741 {"id":"28cb6861-304f-4323-bb9e-ef4e95aed741","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993700} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96de44cf 28cb6861-304f-4323-bb9e-ef4e95aed741 /tidb/cdc/default/default/upstream/7365841790059640291 {"id":7365841790059640291,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28cb6861-304f-4323-bb9e-ef4e95aed741 {"id":"28cb6861-304f-4323-bb9e-ef4e95aed741","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993700} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96de44cf 28cb6861-304f-4323-bb9e-ef4e95aed741 /tidb/cdc/default/default/upstream/7365841790059640291 {"id":7365841790059640291,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28cb6861-304f-4323-bb9e-ef4e95aed741 {"id":"28cb6861-304f-4323-bb9e-ef4e95aed741","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993700} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96de44cf 28cb6861-304f-4323-bb9e-ef4e95aed741 /tidb/cdc/default/default/upstream/7365841790059640291 {"id":7365841790059640291,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/changefeed_pause_resume + sink_uri='file:///tmp/tidb_cdc_test/changefeed_pause_resume/storage_test/ticdc-changefeed-pause-resume-6519?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/changefeed_pause_resume ++ date + echo '[Mon May 6 19:08:23 CST 2024] <<<<<< START storage consumer in changefeed_pause_resume case >>>>>>' [Mon May 6 19:08:23 CST 2024] <<<<<< START storage consumer in changefeed_pause_resume case >>>>>> + cd /tmp/tidb_cdc_test/changefeed_pause_resume + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/changefeed_pause_resume + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/changefeed_pause_resume/storage_test/ticdc-changefeed-pause-resume-6519?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table changefeed_pause_resume.t1 not exists for 1-th check, retry later table changefeed_pause_resume.t1 not exists for 2-th check, retry later check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 4-th time, retry later table changefeed_pause_resume.t1 not exists for 3-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 20.78 secs (179505901 bytes/sec) [Pipeline] { [Pipeline] cache table changefeed_pause_resume.t1 not exists for 4-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:08:32 CST 2024] <<<<<< run test case kafka_simple_handle_key_only_avro success! >>>>>> table changefeed_pause_resume.t1 not exists for 5-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_simple_claim_check/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:08:36 CST 2024] <<<<<< run test case kafka_simple_claim_check success! >>>>>> table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 exists check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 5-th time, retry later wait process 5181 exit for 1-th time... wait process 5181 exit for 2-th time... wait process 5181 exit for 3-th time... wait process 5181 exit for 4-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5181) - No such process wait process 5181 exit for 5-th time... process 5181 already exit [Mon May 6 19:08:35 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.53625364.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:08:38 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365841762574137728 Namespace:default ID:ddl-manager SinkURI:file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-5438?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:08:16.764005643 +0800 CST StartTs:449575307613569027 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001455c20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575307639783429} {CheckpointTs:449575308308250633 MinTableBarrierTs:449575308308250633 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2795c200-f14c-4494-8fad-848ae234dbb3 {"id":"2795c200-f14c-4494-8fad-848ae234dbb3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993716} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96c573b5 2795c200-f14c-4494-8fad-848ae234dbb3 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365841762574137728,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-5438?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:08:16.764005643+08:00","start-ts":449575307613569027,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575307639783429} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449575308439584786,"min-table-barrier-ts":449575308439584786,"admin-job-type":0} /tidb/cdc/default/default/task/position/2795c200-f14c-4494-8fad-848ae234dbb3/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365841762574137728 {"id":7365841762574137728,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365841762574137728 Namespace:default ID:ddl-manager SinkURI:file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-5438?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:08:16.764005643 +0800 CST StartTs:449575307613569027 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001455c20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575307639783429} {CheckpointTs:449575308308250633 MinTableBarrierTs:449575308308250633 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2795c200-f14c-4494-8fad-848ae234dbb3 {"id":"2795c200-f14c-4494-8fad-848ae234dbb3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993716} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96c573b5 2795c200-f14c-4494-8fad-848ae234dbb3 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365841762574137728,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-5438?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:08:16.764005643+08:00","start-ts":449575307613569027,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575307639783429} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449575308439584786,"min-table-barrier-ts":449575308439584786,"admin-job-type":0} /tidb/cdc/default/default/task/position/2795c200-f14c-4494-8fad-848ae234dbb3/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365841762574137728 {"id":7365841762574137728,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365841762574137728 Namespace:default ID:ddl-manager SinkURI:file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-5438?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:08:16.764005643 +0800 CST StartTs:449575307613569027 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001455c20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575307639783429} {CheckpointTs:449575308308250633 MinTableBarrierTs:449575308308250633 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2795c200-f14c-4494-8fad-848ae234dbb3 {"id":"2795c200-f14c-4494-8fad-848ae234dbb3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993716} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96c573b5 2795c200-f14c-4494-8fad-848ae234dbb3 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365841762574137728,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-5438?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:08:16.764005643+08:00","start-ts":449575307613569027,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575307639783429} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449575308439584786,"min-table-barrier-ts":449575308439584786,"admin-job-type":0} /tidb/cdc/default/default/task/position/2795c200-f14c-4494-8fad-848ae234dbb3/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365841762574137728 {"id":7365841762574137728,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:08:38 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.54205422.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:08:39 CST 2024] <<<<<< run test case kafka_simple_claim_check_avro success! >>>>>> check diff failed 1-th time, retry later < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:08:41 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365841762574137728 Namespace:default ID:ddl-manager SinkURI:file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-5438?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:08:16.764005643 +0800 CST StartTs:449575307613569027 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001455c20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575307639783429} {CheckpointTs:449575308439584786 MinTableBarrierTs:449575308439584786 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2795c200-f14c-4494-8fad-848ae234dbb3 {"id":"2795c200-f14c-4494-8fad-848ae234dbb3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993716} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96c573b5 2795c200-f14c-4494-8fad-848ae234dbb3 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365841762574137728,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-5438?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:08:16.764005643+08:00","start-ts":449575307613569027,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575307639783429} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449575308439584786,"min-table-barrier-ts":449575308439584786,"admin-job-type":0} /tidb/cdc/default/default/task/position/2795c200-f14c-4494-8fad-848ae234dbb3/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365841762574137728 {"id":7365841762574137728,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365841762574137728 Namespace:default ID:ddl-manager SinkURI:file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-5438?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:08:16.764005643 +0800 CST StartTs:449575307613569027 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001455c20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575307639783429} {CheckpointTs:449575308439584786 MinTableBarrierTs:449575308439584786 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2795c200-f14c-4494-8fad-848ae234dbb3 {"id":"2795c200-f14c-4494-8fad-848ae234dbb3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993716} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96c573b5 2795c200-f14c-4494-8fad-848ae234dbb3 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365841762574137728,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-5438?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:08:16.764005643+08:00","start-ts":449575307613569027,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575307639783429} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449575308439584786,"min-table-barrier-ts":449575308439584786,"admin-job-type":0} /tidb/cdc/default/default/task/position/2795c200-f14c-4494-8fad-848ae234dbb3/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365841762574137728 {"id":7365841762574137728,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365841762574137728 Namespace:default ID:ddl-manager SinkURI:file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-5438?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:08:16.764005643 +0800 CST StartTs:449575307613569027 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001455c20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575307639783429} {CheckpointTs:449575308439584786 MinTableBarrierTs:449575308439584786 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2795c200-f14c-4494-8fad-848ae234dbb3 {"id":"2795c200-f14c-4494-8fad-848ae234dbb3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993716} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96c573b5 2795c200-f14c-4494-8fad-848ae234dbb3 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365841762574137728,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-5438?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:08:16.764005643+08:00","start-ts":449575307613569027,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575307639783429} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449575308439584786,"min-table-barrier-ts":449575308439584786,"admin-job-type":0} /tidb/cdc/default/default/task/position/2795c200-f14c-4494-8fad-848ae234dbb3/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365841762574137728 {"id":7365841762574137728,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_adapter_compatibility/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:08:42 CST 2024] <<<<<< run test case canal_json_adapter_compatibility success! >>>>>> check diff failed 3-th time, retry later + check_file_exists true + local all_should_exist=true + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/10.data ']' + SINK_URI='file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s' + run_cdc_cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_cleanup.cli.3906.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed.toml Create changefeed successfully! ID: 0f31beb8-8d89-4cad-a070-f5ca87322a6f Info: {"upstream_id":7365841784839173240,"namespace":"default","id":"0f31beb8-8d89-4cad-a070-f5ca87322a6f","sink_uri":"file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s","create_time":"2024-05-06T19:08:42.470483503+08:00","start_ts":449575314358272001,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"cloud_storage_config":{"file_expiration_days":1,"file_cleanup_cron_spec":"* * * * * *"},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575314358272001,"checkpoint_ts":449575314358272001,"checkpoint_time":"2024-05-06 19:08:42.375"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 table ddl_manager.finish_mark not exists for 3-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_basic/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:08:45 CST 2024] <<<<<< run test case canal_json_basic success! >>>>>> check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 6-th time, retry later check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 4-th check, retry later check diff failed 5-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_content_compatible/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:08:49 CST 2024] <<<<<< run test case canal_json_content_compatible success! >>>>>> table ddl_manager.finish_mark not exists for 5-th check, retry later check diff failed 6-th time, retry later table ddl_manager.finish_mark not exists for 6-th check, retry later check diff failed 7-th time, retry later table ddl_manager.finish_mark not exists for 7-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/multi_topics/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:08:53 CST 2024] <<<<<< run test case multi_topics success! >>>>>> check diff successfully table ddl_manager.finish_mark not exists for 8-th check, retry later table ddl_manager.finish_mark not exists for 9-th check, retry later check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 7-th time, retry later check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 10-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/avro_basic/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:09:00 CST 2024] <<<<<< run test case avro_basic success! >>>>>> check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 11-th check, retry later check diff failed 3-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:09:03 CST 2024] <<<<<< run test case canal_json_handle_key_only success! >>>>>> table ddl_manager.finish_mark not exists for 12-th check, retry later check diff failed 4-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 32.54 secs (114644369 bytes/sec) [Pipeline] { [Pipeline] cache table ddl_manager.finish_mark not exists for 13-th check, retry later + check_file_exists false + local all_should_exist=false + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-05/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-05/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-05/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-04/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-04/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-04/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-04/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-04/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-04/10.data ']' + run_sql_file /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/data/schema.sql 127.0.0.1 4000 + run_sql_file /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/data/data.sql 127.0.0.1 4000 + run_storage_consumer /tmp/tidb_cdc_test/storage_cleanup 'file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed.toml '' + workdir=/tmp/tidb_cdc_test/storage_cleanup + sink_uri='file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Mon May 6 19:09:05 CST 2024] <<<<<< START storage consumer in storage_cleanup case >>>>>>' [Mon May 6 19:09:05 CST 2024] <<<<<< START storage consumer in storage_cleanup case >>>>>> + cd /tmp/tidb_cdc_test/storage_cleanup + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/storage_cleanup + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/storage_cleanup/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed.toml + sleep 8 check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 14-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:09:07 CST 2024] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> check diff failed 6-th time, retry later table ddl_manager.finish_mark not exists for 15-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_claim_check/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:09:10 CST 2024] <<<<<< run test case canal_json_claim_check success! >>>>>> check diff failed 7-th time, retry later table ddl_manager.finish_mark not exists for 16-th check, retry later check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl [2024/05/06 19:08:10.658 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575305119793165,\"CommitTs\":449575305119793166,\"RowID\":1,\"PhysicalTableID\":106,\"TableInfo\":{\"id\":106,\"name\":{\"O\":\"example1\",\"L\":\"example1\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575305093316642,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example1\",\"TableID\":106,\"IsPartition\":false},\"Version\":449575305106685953,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449575306024189961,\"HandleKey\":1}"] [2024/05/06 19:08:12.657 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575305211281425,\"CommitTs\":449575305211281427,\"RowID\":1,\"PhysicalTableID\":109,\"TableInfo\":{\"id\":109,\"name\":{\"O\":\"example2\",\"L\":\"example2\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575305198174242,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example2\",\"TableID\":109,\"IsPartition\":false},\"Version\":449575305198174248,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449575306535108653,\"HandleKey\":1}"] [2024/05/06 19:08:14.657 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575305316139015,\"CommitTs\":449575305316139017,\"RowID\":1,\"PhysicalTableID\":112,\"TableInfo\":{\"id\":112,\"name\":{\"O\":\"example3\",\"L\":\"example3\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":7,\"Flag\":128,\"Flen\":19,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575305290186784,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example3\",\"TableID\":112,\"IsPartition\":false},\"Version\":449575305290186790,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":\"2023-04-19 11:48:00\"}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":36,\"SplitTxn\":false,\"ReplicatingTs\":449575307072765958,\"HandleKey\":1}"] [2024/05/06 19:08:16.656 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575305408151566,\"CommitTs\":449575305408151568,\"RowID\":1,\"PhysicalTableID\":115,\"TableInfo\":{\"id\":115,\"name\":{\"O\":\"example4\",\"L\":\"example4\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":256,\"Decimal\":0,\"Charset\":\"utf8mb4\",\"Collate\":\"utf8mb4_bin\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575305395044356,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example4\",\"TableID\":115,\"IsPartition\":false},\"Version\":449575305395044362,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":64},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":\"MjAyMy0wNC0xOSAxMTo0ODowMA==\"}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":47,\"SplitTxn\":false,\"ReplicatingTs\":449575307597053958,\"HandleKey\":1}"] [2024/05/06 19:08:18.657 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575305513009159,\"CommitTs\":449575305513009161,\"RowID\":1,\"PhysicalTableID\":118,\"TableInfo\":{\"id\":118,\"name\":{\"O\":\"example5\",\"L\":\"example5\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575305486532653,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example5\",\"TableID\":118,\"IsPartition\":false},\"Version\":449575305486532658,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":-1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449575308121079818,\"HandleKey\":1}"] [2024/05/06 19:08:20.656 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575305604759556,\"CommitTs\":449575305604759557,\"RowID\":1,\"PhysicalTableID\":121,\"TableInfo\":{\"id\":121,\"name\":{\"O\":\"example6\",\"L\":\"example6\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575305578545183,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example6\",\"TableID\":121,\"IsPartition\":false},\"Version\":449575305578545187,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":-1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449575308645367815,\"HandleKey\":1}"] [2024/05/06 19:08:22.657 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575305683402769,\"CommitTs\":449575305683402771,\"RowID\":1,\"PhysicalTableID\":124,\"TableInfo\":{\"id\":124,\"name\":{\"O\":\"example7\",\"L\":\"example7\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":256,\"Decimal\":0,\"Charset\":\"utf8\",\"Collate\":\"utf8_general_ci\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575305670295581,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example7\",\"TableID\":124,\"IsPartition\":false},\"Version\":449575305670295586,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":64},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":\"MjAyMy0wNC0xOSAxMTo0ODowMA==\"}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":47,\"SplitTxn\":false,\"ReplicatingTs\":449575309169655814,\"HandleKey\":1}"] [2024/05/06 19:08:24.657 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575305735569417,\"CommitTs\":449575305735569418,\"RowID\":-1,\"PhysicalTableID\":128,\"TableInfo\":{\"id\":127,\"name\":{\"O\":\"example8\",\"L\":\"example8\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575305709355040,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`id`\",\"columns\":[],\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":128,\"name\":{\"O\":\"b0\",\"L\":\"b0\"},\"less_than\":[\"0\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":129,\"name\":{\"O\":\"b1\",\"L\":\"b1\"},\"less_than\":[\"MAXVALUE\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":[],\"dropping_definitions\":[],\"NewPartitionIDs\":null,\"states\":null,\"num\":2,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":0,\"ddl_expr\":\"\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example8\",\"TableID\":127,\"IsPartition\":true},\"Version\":449575305709355046,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":-1},{\"column_id\":2,\"value\":-1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449575309694205962,\"HandleKey\":-1}"] [2024/05/06 19:08:24.657 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575305735569419,\"CommitTs\":449575305735569420,\"RowID\":1,\"PhysicalTableID\":129,\"TableInfo\":{\"id\":127,\"name\":{\"O\":\"example8\",\"L\":\"example8\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575305709355040,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`id`\",\"columns\":[],\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":128,\"name\":{\"O\":\"b0\",\"L\":\"b0\"},\"less_than\":[\"0\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":129,\"name\":{\"O\":\"b1\",\"L\":\"b1\"},\"less_than\":[\"MAXVALUE\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":[],\"dropping_definitions\":[],\"NewPartitionIDs\":null,\"states\":null,\"num\":2,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":0,\"ddl_expr\":\"\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example8\",\"TableID\":127,\"IsPartition\":true},\"Version\":449575305709355046,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449575309694205963,\"HandleKey\":1}"] [2024/05/06 19:08:26.656 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575305801367568,\"CommitTs\":449575305801367569,\"RowID\":1,\"PhysicalTableID\":132,\"TableInfo\":{\"id\":132,\"name\":{\"O\":\"example9\",\"L\":\"example9\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":256,\"Decimal\":0,\"Charset\":\"utf8mb4\",\"Collate\":\"utf8mb4_bin\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575305787998242,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example9\",\"TableID\":132,\"IsPartition\":false},\"Version\":449575305787998247,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":64},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":\"MjAyMy0wNC0xOSAxMTo0ODowMA==\"}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":47,\"SplitTxn\":false,\"ReplicatingTs\":449575310218493958,\"HandleKey\":1}"] [2024/05/06 19:08:28.657 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575305853534215,\"CommitTs\":449575305853534216,\"RowID\":1,\"PhysicalTableID\":135,\"TableInfo\":{\"id\":135,\"name\":{\"O\":\"example10\",\"L\":\"example10\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":5,\"Flag\":0,\"Flen\":22,\"Decimal\":-1,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575305827581983,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example10\",\"TableID\":135,\"IsPartition\":false},\"Version\":449575305827581987,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":36,\"SplitTxn\":false,\"ReplicatingTs\":449575310742781958,\"HandleKey\":1}"] [2024/05/06 19:08:30.657 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575305945284623,\"CommitTs\":449575305945284625,\"RowID\":1,\"PhysicalTableID\":138,\"TableInfo\":{\"id\":138,\"name\":{\"O\":\"example11\",\"L\":\"example11\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":8,\"Flag\":4099,\"Flen\":20,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":8,\"Flag\":0,\"Flen\":20,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575305932177426,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example11\",\"TableID\":138,\"IsPartition\":false},\"Version\":449575305932177432,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449575311266807820,\"HandleKey\":1}"] [2024/05/06 19:08:32.657 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575306050142223,\"CommitTs\":449575306050142225,\"RowID\":1,\"PhysicalTableID\":141,\"TableInfo\":{\"id\":141,\"name\":{\"O\":\"example12\",\"L\":\"example12\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":251,\"Flag\":0,\"Flen\":4294967295,\"Decimal\":0,\"Charset\":\"utf8mb4\",\"Collate\":\"utf8mb4_bin\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575306037035019,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example12\",\"TableID\":141,\"IsPartition\":false},\"Version\":449575306037035023,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":64},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":\"MjAyMy0wNC0xOSAxMTo0ODowMA==\"}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":47,\"SplitTxn\":false,\"ReplicatingTs\":449575311791095814,\"HandleKey\":1}"] [2024/05/06 19:08:34.657 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575306142154769,\"CommitTs\":449575306142154771,\"RowID\":1,\"PhysicalTableID\":144,\"TableInfo\":{\"id\":144,\"name\":{\"O\":\"example13\",\"L\":\"example13\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":247,\"Flag\":0,\"Flen\":1,\"Decimal\":0,\"Charset\":\"utf8mb4\",\"Collate\":\"utf8mb4_bin\",\"Elems\":[\"a\",\"b\",\"c\"],\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575306129047584,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example13\",\"TableID\":144,\"IsPartition\":false},\"Version\":449575306129047588,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":64},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449575312315383814,\"HandleKey\":1}"] [2024/05/06 19:08:36.657 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575306246750223,\"CommitTs\":449575306246750225,\"RowID\":1,\"PhysicalTableID\":147,\"TableInfo\":{\"id\":147,\"name\":{\"O\":\"example14\",\"L\":\"example14\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":256,\"Decimal\":0,\"Charset\":\"utf8mb4\",\"Collate\":\"utf8mb4_bin\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575306220797988,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example14\",\"TableID\":147,\"IsPartition\":false},\"Version\":449575306233643012,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":64},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":\"MjAyMy0wNC0xOSAxMTo0ODowMA==\"}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":47,\"SplitTxn\":false,\"ReplicatingTs\":449575312839671818,\"HandleKey\":1}"] [2024/05/06 19:08:38.657 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575306351607811,\"CommitTs\":449575306351607813,\"RowID\":1,\"PhysicalTableID\":150,\"TableInfo\":{\"id\":150,\"name\":{\"O\":\"example15\",\"L\":\"example15\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575306325655584,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example15\",\"TableID\":150,\"IsPartition\":false},\"Version\":449575306325655588,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":-1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449575313363959814,\"HandleKey\":1}"] [2024/05/06 19:08:40.658 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575306443620368,\"CommitTs\":449575306443620369,\"RowID\":1,\"PhysicalTableID\":154,\"TableInfo\":{\"id\":153,\"name\":{\"O\":\"example16\",\"L\":\"example16\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"a\",\"L\":\"a\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575306430251022,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`a`\",\"columns\":[],\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":154,\"name\":{\"O\":\"p0\",\"L\":\"p0\"},\"less_than\":[\"6\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":155,\"name\":{\"O\":\"p1\",\"L\":\"p1\"},\"less_than\":[\"11\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":156,\"name\":{\"O\":\"p2\",\"L\":\"p2\"},\"less_than\":[\"21\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":[],\"dropping_definitions\":[],\"NewPartitionIDs\":null,\"states\":null,\"num\":3,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":0,\"ddl_expr\":\"\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example16\",\"TableID\":153,\"IsPartition\":true},\"Version\":449575306430251026,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449575313888247815,\"HandleKey\":1}"] [2024/05/06 19:08:42.657 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575306469572621,\"CommitTs\":449575306469572622,\"RowID\":18,\"PhysicalTableID\":158,\"TableInfo\":{\"id\":158,\"name\":{\"O\":\"example17\",\"L\":\"example17\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"a\",\"L\":\"a\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575306443620387,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example17\",\"TableID\":158,\"IsPartition\":false},\"Version\":449575306443620391,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":18}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449575314412535818,\"HandleKey\":18}"] [2024/05/06 19:08:44.657 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575306522001424,\"CommitTs\":449575306522001426,\"RowID\":1,\"PhysicalTableID\":162,\"TableInfo\":{\"id\":161,\"name\":{\"O\":\"example18\",\"L\":\"example18\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575306508894237,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`id`\",\"columns\":[],\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":162,\"name\":{\"O\":\"pBefore1950\",\"L\":\"pbefore1950\"},\"less_than\":[\"1950\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":163,\"name\":{\"O\":\"p1950\",\"L\":\"p1950\"},\"less_than\":[\"1960\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":164,\"name\":{\"O\":\"p1960\",\"L\":\"p1960\"},\"less_than\":[\"1970\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":165,\"name\":{\"O\":\"p1970\",\"L\":\"p1970\"},\"less_than\":[\"1980\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":166,\"name\":{\"O\":\"p1980\",\"L\":\"p1980\"},\"less_than\":[\"1990\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":167,\"name\":{\"O\":\"p1990\",\"L\":\"p1990\"},\"less_than\":[\"2000\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":[],\"dropping_definitions\":[],\"NewPartitionIDs\":null,\"states\":null,\"num\":6,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":0,\"ddl_expr\":\"\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example18\",\"TableID\":161,\"IsPartition\":true},\"Version\":449575306508894244,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449575314936823814,\"HandleKey\":1}"] [2024/05/06 19:08:44.658 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575306522001424,\"CommitTs\":449575306522001426,\"RowID\":1977,\"PhysicalTableID\":165,\"TableInfo\":{\"id\":161,\"name\":{\"O\":\"example18\",\"L\":\"example18\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575306508894237,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`id`\",\"columns\":[],\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":162,\"name\":{\"O\":\"pBefore1950\",\"L\":\"pbefore1950\"},\"less_than\":[\"1950\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":163,\"name\":{\"O\":\"p1950\",\"L\":\"p1950\"},\"less_than\":[\"1960\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":164,\"name\":{\"O\":\"p1960\",\"L\":\"p1960\"},\"less_than\":[\"1970\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":165,\"name\":{\"O\":\"p1970\",\"L\":\"p1970\"},\"less_than\":[\"1980\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":166,\"name\":{\"O\":\"p1980\",\"L\":\"p1980\"},\"less_than\":[\"1990\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":167,\"name\":{\"O\":\"p1990\",\"L\":\"p1990\"},\"less_than\":[\"2000\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":[],\"dropping_definitions\":[],\"NewPartitionIDs\":null,\"states\":null,\"num\":6,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":0,\"ddl_expr\":\"\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example18\",\"TableID\":161,\"IsPartition\":true},\"Version\":449575306508894244,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1977}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449575314936823817,\"HandleKey\":1977}"] [2024/05/06 19:08:45.657 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575306522001424,\"CommitTs\":449575306522001426,\"RowID\":1999,\"PhysicalTableID\":167,\"TableInfo\":{\"id\":161,\"name\":{\"O\":\"example18\",\"L\":\"example18\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575306508894237,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`id`\",\"columns\":[],\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":162,\"name\":{\"O\":\"pBefore1950\",\"L\":\"pbefore1950\"},\"less_than\":[\"1950\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":163,\"name\":{\"O\":\"p1950\",\"L\":\"p1950\"},\"less_than\":[\"1960\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":164,\"name\":{\"O\":\"p1960\",\"L\":\"p1960\"},\"less_than\":[\"1970\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":165,\"name\":{\"O\":\"p1970\",\"L\":\"p1970\"},\"less_than\":[\"1980\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":166,\"name\":{\"O\":\"p1980\",\"L\":\"p1980\"},\"less_than\":[\"1990\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":167,\"name\":{\"O\":\"p1990\",\"L\":\"p1990\"},\"less_than\":[\"2000\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":[],\"dropping_definitions\":[],\"NewPartitionIDs\":null,\"states\":null,\"num\":6,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":0,\"ddl_expr\":\"\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example18\",\"TableID\":161,\"IsPartition\":true},\"Version\":449575306508894244,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1999}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449575315199229965,\"HandleKey\":1999}"] [2024/05/06 19:08:51.757 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575306745085960,\"CommitTs\":449575306745085961,\"RowID\":2003,\"PhysicalTableID\":174,\"TableInfo\":{\"id\":161,\"name\":{\"O\":\"example18\",\"L\":\"example18\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":null,\"constraint_info\":null,\"fk_info\":null,\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575306718871560,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`id`\",\"columns\":null,\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":169,\"name\":{\"O\":\"pBefore1960\",\"L\":\"pbefore1960\"},\"less_than\":[\"1960\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":164,\"name\":{\"O\":\"p1960\",\"L\":\"p1960\"},\"less_than\":[\"1970\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":165,\"name\":{\"O\":\"p1970\",\"L\":\"p1970\"},\"less_than\":[\"1980\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":172,\"name\":{\"O\":\"p1980\",\"L\":\"p1980\"},\"less_than\":[\"1990\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":174,\"name\":{\"O\":\"p1990to2010\",\"L\":\"p1990to2010\"},\"less_than\":[\"2010\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":null,\"dropping_definitions\":null,\"NewPartitionIDs\":null,\"states\":null,\"num\":5,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":1,\"ddl_expr\":\"`id`\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example18\",\"TableID\":161,\"IsPartition\":true},\"Version\":449575306718871565,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":2003}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449575316785201157,\"HandleKey\":2003}"] [2024/05/06 19:08:55.757 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449575307321540627,\"CommitTs\":449575307321540628,\"RowID\":1,\"PhysicalTableID\":182,\"TableInfo\":{\"id\":182,\"name\":{\"O\":\"example19\",\"L\":\"example19\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"a\",\"L\":\"a\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449575307295588391,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example19\",\"TableID\":182,\"IsPartition\":false},\"Version\":449575307308695553,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449575317833777157,\"HandleKey\":1}"] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... check diff failed 8-th time, retry later table ddl_manager.finish_mark not exists for 17-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:09:13 CST 2024] <<<<<< run test case lossy_ddl success! >>>>>> + check_sync_diff /tmp/tidb_cdc_test/storage_cleanup /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/diff_config.toml 100 check diff failed 1-th time, retry later check diff successfully Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 7.43 secs (501798214 bytes/sec) [Pipeline] { [Pipeline] cache table ddl_manager.finish_mark not exists for 18-th check, retry later check diff successfully + check_logs /tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Mon May 6 19:09:16 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>>' [Mon May 6 19:09:16 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>> + stop_tidb_cluster =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/open_protocol_claim_check/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:09:17 CST 2024] <<<<<< run test case open_protocol_claim_check success! >>>>>> table ddl_manager.finish_mark not exists for 19-th check, retry later check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 20-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_basic/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 21-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 22-th check, retry later check diff failed 3-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_basic Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 23-th check, retry later check diff failed 4-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/storage_csv_update Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 24-th check, retry later check diff failed 5-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 25-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 6-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 26-th check, retry later check diff failed 7-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 27-th check, retry later check diff failed 8-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 28-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 18.18 secs (205211523 bytes/sec) [Pipeline] { [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36610a9c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55, pid:6713, start at 2024-05-06 19:09:35.279638316 +0800 CST m=+5.314063329 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:35.288 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:35.271 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:35.271 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully start tidb cluster in /tmp/tidb_cdc_test/csv_storage_basic Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 29-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36610a9c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55, pid:6713, start at 2024-05-06 19:09:35.279638316 +0800 CST m=+5.314063329 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:35.288 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:35.271 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:35.271 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36610cec0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55, pid:6785, start at 2024-05-06 19:09:35.460456448 +0800 CST m=+5.414864047 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:35.471 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:35.469 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:35.469 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_storage_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_storage_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_storage_basic/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_storage_basic/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_storage_basic/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366139d8000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-mnw-vn0pb, pid:4954, start at 2024-05-06 19:09:38.303726272 +0800 CST m=+5.521875604 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:38.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:38.294 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:38.294 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 30-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 1-th time, retry later [Mon May 6 19:09:40 CST 2024] <<<<<< START cdc server in canal_json_storage_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_basic.80298031.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_basic/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366139d8000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-mnw-vn0pb, pid:4954, start at 2024-05-06 19:09:38.303726272 +0800 CST m=+5.521875604 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:38.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:38.294 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:38.294 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36613b900003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-mnw-vn0pb, pid:5035, start at 2024-05-06 19:09:38.405604596 +0800 CST m=+5.557379143 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:38.415 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:38.404 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:38.404 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/storage_csv_update/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/storage_csv_update/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/storage_csv_update/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/storage_csv_update/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/storage_csv_update/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 31-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [Mon May 6 19:09:43 CST 2024] <<<<<< START cdc server in storage_csv_update case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_csv_update.62896291.out server --log-file /tmp/tidb_cdc_test/storage_csv_update/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/storage_csv_update/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:09:43 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fb81de06-5ecf-4cd7-987b-b7bcc7ee862d {"id":"fb81de06-5ecf-4cd7-987b-b7bcc7ee862d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993780} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98180dcc fb81de06-5ecf-4cd7-987b-b7bcc7ee862d /tidb/cdc/default/default/upstream/7365842141338675266 {"id":7365842141338675266,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fb81de06-5ecf-4cd7-987b-b7bcc7ee862d {"id":"fb81de06-5ecf-4cd7-987b-b7bcc7ee862d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993780} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98180dcc fb81de06-5ecf-4cd7-987b-b7bcc7ee862d /tidb/cdc/default/default/upstream/7365842141338675266 {"id":7365842141338675266,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fb81de06-5ecf-4cd7-987b-b7bcc7ee862d {"id":"fb81de06-5ecf-4cd7-987b-b7bcc7ee862d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993780} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98180dcc fb81de06-5ecf-4cd7-987b-b7bcc7ee862d /tidb/cdc/default/default/upstream/7365842141338675266 {"id":7365842141338675266,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_basic.cli.8087.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml Create changefeed successfully! ID: c8759220-30fb-48ba-8cd7-a8ae1211726b Info: {"upstream_id":7365842141338675266,"namespace":"default","id":"c8759220-30fb-48ba-8cd7-a8ae1211726b","sink_uri":"file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:09:43.844356195+08:00","start_ts":449575330452865026,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575330452865026,"checkpoint_ts":449575330452865026,"checkpoint_time":"2024-05-06 19:09:43.771"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 32-th check, retry later check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + workdir=/tmp/tidb_cdc_test/canal_json_storage_basic + sink_uri='file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/canal_json_storage_basic ++ date + echo '[Mon May 6 19:09:45 CST 2024] <<<<<< START storage consumer in canal_json_storage_basic case >>>>>>' [Mon May 6 19:09:45 CST 2024] <<<<<< START storage consumer in canal_json_storage_basic case >>>>>> + cd /tmp/tidb_cdc_test/canal_json_storage_basic + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/canal_json_storage_basic + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/canal_json_storage_basic/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml table ddl_manager.finish_mark not exists for 33-th check, retry later check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:09:46 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a02120a0-e875-4b86-80eb-0a0f05644478 {"id":"a02120a0-e875-4b86-80eb-0a0f05644478","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993783} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d982314cf a02120a0-e875-4b86-80eb-0a0f05644478 /tidb/cdc/default/default/upstream/7365842154980006574 {"id":7365842154980006574,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a02120a0-e875-4b86-80eb-0a0f05644478 {"id":"a02120a0-e875-4b86-80eb-0a0f05644478","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993783} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d982314cf a02120a0-e875-4b86-80eb-0a0f05644478 /tidb/cdc/default/default/upstream/7365842154980006574 {"id":7365842154980006574,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a02120a0-e875-4b86-80eb-0a0f05644478 {"id":"a02120a0-e875-4b86-80eb-0a0f05644478","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993783} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d982314cf a02120a0-e875-4b86-80eb-0a0f05644478 /tidb/cdc/default/default/upstream/7365842154980006574 {"id":7365842154980006574,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_csv_update.cli.6342.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml Create changefeed successfully! ID: ae2c0c47-2863-4e98-a4c9-33c1ded440a0 Info: {"upstream_id":7365842154980006574,"namespace":"default","id":"ae2c0c47-2863-4e98-a4c9-33c1ded440a0","sink_uri":"file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:09:46.644960436+08:00","start_ts":449575331180052485,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575331180052485,"checkpoint_ts":449575331180052485,"checkpoint_time":"2024-05-06 19:09:46.545"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 7.32 secs (509801190 bytes/sec) [Pipeline] { [Pipeline] cache table ddl_manager.finish_mark not exists for 34-th check, retry later + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 35-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3661d3680008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9, pid:4926, start at 2024-05-06 19:09:48.132318349 +0800 CST m=+6.161728219 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:48.141 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:48.122 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:48.122 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3661d3680008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9, pid:4926, start at 2024-05-06 19:09:48.132318349 +0800 CST m=+6.161728219 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:48.141 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:48.122 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:48.122 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3661d6640002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9, pid:4961, start at 2024-05-06 19:09:48.314145507 +0800 CST m=+6.232355344 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:48.322 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:48.313 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:48.313 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/csv_storage_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/csv_storage_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/csv_storage_basic/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/csv_storage_basic/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/csv_storage_basic/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 6-th time, retry later table ddl_manager.finish_mark not exists for 36-th check, retry later [Mon May 6 19:09:52 CST 2024] <<<<<< START cdc server in csv_storage_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_basic.63256327.out server --log-file /tmp/tidb_cdc_test/csv_storage_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_basic/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 7-th time, retry later check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 37-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 5.78 secs (644888418 bytes/sec) [Pipeline] { [Pipeline] sh check diff successfully [Pipeline] sh + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:09:55 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50e33091-2bff-42b5-87d1-517b9bc7b9d7 {"id":"50e33091-2bff-42b5-87d1-517b9bc7b9d7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993792} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9848a8cd 50e33091-2bff-42b5-87d1-517b9bc7b9d7 /tidb/cdc/default/default/upstream/7365842191149614586 {"id":7365842191149614586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50e33091-2bff-42b5-87d1-517b9bc7b9d7 {"id":"50e33091-2bff-42b5-87d1-517b9bc7b9d7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993792} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9848a8cd 50e33091-2bff-42b5-87d1-517b9bc7b9d7 /tidb/cdc/default/default/upstream/7365842191149614586 {"id":7365842191149614586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50e33091-2bff-42b5-87d1-517b9bc7b9d7 {"id":"50e33091-2bff-42b5-87d1-517b9bc7b9d7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993792} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9848a8cd 50e33091-2bff-42b5-87d1-517b9bc7b9d7 /tidb/cdc/default/default/upstream/7365842191149614586 {"id":7365842191149614586,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_basic.cli.6374.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml [Pipeline] sh table ddl_manager.finish_mark not exists for 38-th check, retry later [Pipeline] sh Create changefeed successfully! ID: a14b2393-0c36-4e89-bcf5-dd16e8260dad Info: {"upstream_id":7365842191149614586,"namespace":"default","id":"a14b2393-0c36-4e89-bcf5-dd16e8260dad","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s","create_time":"2024-05-06T19:09:55.987565064+08:00","start_ts":449575333625069571,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575333625069571,"checkpoint_ts":449575333625069571,"checkpoint_time":"2024-05-06 19:09:55.872"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check diff failed 2-th time, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G10 Run cases: default_value simple cdc_server_tips event_filter sql_mode PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=ea3eff86-f747-4c97-a9d2-68070e85e2d6 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-c73tj GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1816-c73tj pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:09:56 CST 2024] <<<<<< run test case default_value success! >>>>>> [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G08 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=cfc41d59-4a7d-4530-8673-198ead2cd5c2 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-c8j-cs84c HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-c8jq9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1816-c8j-cs84c pingcap_tiflow_pull_cdc_integration_storage_test_1816-c8jq9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-c8j-cs84c GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: storage... <<================= Run cases: processor_err_chan changefeed_reconstruct multi_capture synced_status_with_redo PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=a0505547-ba5a-427c-96a1-8730334774eb BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-bpttb GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1816-bpttb pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G04 Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=4b3986e2-baf8-4bd5-b60a-f1c05295392d BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-sb43l GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m pingcap_tiflow_pull_cdc_integration_storage_test_1816-sb43l GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G13 Run cases: tiflash region_merge common_1 PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=033ff8da-4c6e-475d-92a0-4efa78baa5ff BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-xp8-cvcxs HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-xp897 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1816-xp8-cvcxs pingcap_tiflow_pull_cdc_integration_storage_test_1816-xp897 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-xp8-cvcxs GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: storage... <<================= [Pipeline] sh The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G12 Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=b92d2dc2-7db8-4f9d-8175-2a55585e9076 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-j5h-9wmdc HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-j5hkm GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1816-j5h-9wmdc pingcap_tiflow_pull_cdc_integration_storage_test_1816-j5hkm GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-j5h-9wmdc GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:09:57 CST 2024] <<<<<< run test case many_pk_or_uk success! >>>>>> [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G06 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G03 Run cases: row_format drop_many_tables processor_stop_delay partition_table PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=e2732230-53f8-463a-b63b-890b9d8dd6c5 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-404l8 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5 pingcap_tiflow_pull_cdc_integration_storage_test_1816-404l8 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 39-th check, retry later Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=05fb4753-a8c3-45fc-8945-fe5b0567833c BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-pfn35 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1816-pfn35 pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... + set +x check diff successfully [Mon May 6 19:09:57 CST 2024] <<<<<< run test case canal_json_storage_basic success! >>>>>> + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=2989777d-4b82-41eb-887b-704f7bb650c1 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-2p4-slmd8 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-2p408 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1816-2p408 pingcap-tiflow-pull-cdc-integration-storage-test-1816-2p4-slmd8 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-2p4-slmd8 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... check diff failed 1-th time, retry later + workdir=/tmp/tidb_cdc_test/csv_storage_basic + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_basic ++ date + echo '[Mon May 6 19:09:59 CST 2024] <<<<<< START storage consumer in csv_storage_basic case >>>>>>' [Mon May 6 19:09:59 CST 2024] <<<<<< START storage consumer in csv_storage_basic case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_basic + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_basic + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_basic/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml table ddl_manager.finish_mark not exists for 40-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... The 1 times to try to start tidb cluster... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... check diff failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 41-th check, retry later Verifying downstream PD is started... check diff failed 3-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 42-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... Verifying downstream PD is started... check diff failed 4-th time, retry later Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv table ddl_manager.finish_mark not exists for 43-th check, retry later [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 44-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... [Pipeline] container [Pipeline] { ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 6-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 45-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 7-th time, retry later [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366336a4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m, pid:1285, start at 2024-05-06 19:10:10.868004169 +0800 CST m=+5.237811654 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:10.877 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:10.857 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:10.857 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] { table ddl_manager.finish_mark not exists for 46-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] cache check diff successfully [Mon May 6 19:10:12 CST 2024] <<<<<< run test case csv_storage_basic success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366351580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:1223, start at 2024-05-06 19:10:12.594636944 +0800 CST m=+5.425864860 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:12.602 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:12.566 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:12.566 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 8-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 47-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366336a4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m, pid:1285, start at 2024-05-06 19:10:10.868004169 +0800 CST m=+5.237811654 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:10.877 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:10.857 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:10.857 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366337f00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m, pid:1368, start at 2024-05-06 19:10:10.988500051 +0800 CST m=+5.298291137 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:10.995 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:10.990 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:10.990 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366357e40006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-xp8-cvcxs, pid:1160, start at 2024-05-06 19:10:12.989623953 +0800 CST m=+5.804958278 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:12.998 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:12.985 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:12.985 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36636150000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-j5h-9wmdc, pid:1350, start at 2024-05-06 19:10:13.597985737 +0800 CST m=+5.464366316 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:13.607 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:13.588 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:13.588 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36634fa40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-2p4-slmd8, pid:1240, start at 2024-05-06 19:10:12.474604208 +0800 CST m=+5.523897254 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:12.481 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:12.457 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:12.457 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36634fa40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-2p4-slmd8, pid:1240, start at 2024-05-06 19:10:12.474604208 +0800 CST m=+5.523897254 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:12.481 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:12.457 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:12.457 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366351200014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-2p4-slmd8, pid:1314, start at 2024-05-06 19:10:12.575621258 +0800 CST m=+5.548832877 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:12.583 +0800 Current GC worker leader lease. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tidb_server_version 196 Bootstrap version. Do not delete. tikv_gc_enable true Current GC enable status system_tz Asia/Shanghai TiDB Global System Timezone. tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_leader_uuid 63d366351580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_last_run_time 20240506-19:10:12.552 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:12.552 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:1223, start at 2024-05-06 19:10:12.594636944 +0800 CST m=+5.425864860 Host name and pid of current GC leader. (DO NOT EDIT) Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community tikv_gc_leader_lease 20240506-19:12:12.602 +0800 Current GC worker leader lease. (DO NOT EDIT) Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto tikv_gc_enable true Current GC enable status Profile: RELWITHDEBINFO tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. Compiler: clang++ 13.0.0 Raft Proxy tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD tikv_gc_last_run_time 20240506-19:10:12.566 +0800 The time when last GC starts. (DO NOT EDIT) UTC Build Time: 2024-05-06 04:09:34 tikv_gc_safe_point 20240506-19:00:12.566 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT Prometheus Prefix: tiflash_proxy_ Profile: release bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366351540015 Current GC worker leader UUID. (DO NOT EDIT) Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:1298, start at 2024-05-06 19:10:12.609394623 +0800 CST m=+5.370479636 Host name and pid of current GC leader. (DO NOT EDIT) Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log tikv_gc_leader_lease 20240506-19:12:12.616 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:12.615 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:12.615 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366376a40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c8j-cs84c, pid:1360, start at 2024-05-06 19:10:14.992029747 +0800 CST m=+5.991200368 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:14.999 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:15.003 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:15.003 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 48-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36635b580018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5, pid:1306, start at 2024-05-06 19:10:13.240308321 +0800 CST m=+5.501945799 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:13.248 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:13.206 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:13.206 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36635b580018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5, pid:1306, start at 2024-05-06 19:10:13.240308321 +0800 CST m=+5.501945799 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:13.248 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:13.206 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:13.206 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36636af0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5, pid:1375, start at 2024-05-06 19:10:14.214921135 +0800 CST m=+6.423601828 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:14.223 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:14.204 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:14.204 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2614.out cli tso query --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366357e40006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-xp8-cvcxs, pid:1160, start at 2024-05-06 19:10:12.989623953 +0800 CST m=+5.804958278 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:12.998 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:12.985 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:12.985 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366358dc000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-xp8-cvcxs, pid:1238, start at 2024-05-06 19:10:13.068784878 +0800 CST m=+5.812971493 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:13.076 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:13.047 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:13.047 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36636150000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-j5h-9wmdc, pid:1350, start at 2024-05-06 19:10:13.597985737 +0800 CST m=+5.464366316 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:13.607 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:13.588 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:13.588 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366361280015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-j5h-9wmdc, pid:1429, start at 2024-05-06 19:10:13.61414908 +0800 CST m=+5.415983180 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:13.624 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:13.628 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:13.628 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 19:10:17 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.26292631.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366376a40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c8j-cs84c, pid:1360, start at 2024-05-06 19:10:14.992029747 +0800 CST m=+5.991200368 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:14.999 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:15.003 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:15.003 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366376ac0011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c8j-cs84c, pid:1441, start at 2024-05-06 19:10:14.988316946 +0800 CST m=+5.908220725 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:14.996 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:14.955 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:14.955 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36638cc0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms, pid:1163, start at 2024-05-06 19:10:16.386034107 +0800 CST m=+7.977499482 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:16.395 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:16.368 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:16.368 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36638cc0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms, pid:1163, start at 2024-05-06 19:10:16.386034107 +0800 CST m=+7.977499482 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:16.395 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:16.368 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:16.368 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366385a4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms, pid:1225, start at 2024-05-06 19:10:15.937700149 +0800 CST m=+7.456508715 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:15.948 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:15.913 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:15.913 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='449575338981982210 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575338981982210 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:10:17 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.26492651.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 49-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2634.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2614.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.2700.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2697.out cli tso query --pd=http://127.0.0.1:2379 [Mon May 6 19:10:19 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/ProcessorAddTableError=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.27562758.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2588.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575339545329665 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575339545329665 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:10:19 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.26802682.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3663b32c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:1373, start at 2024-05-06 19:10:18.851014243 +0800 CST m=+5.267971804 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:18.860 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:18.827 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:18.827 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ls /tmp/tidb_cdc_test/storage_csv_update/storage_test/test/test_update 449575331599482903 run task successfully start check update + workdir=/tmp/tidb_cdc_test/storage_csv_update + sink_uri='file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/storage_csv_update ++ date + echo '[Mon May 6 19:10:18 CST 2024] <<<<<< START storage consumer in storage_csv_update case >>>>>>' [Mon May 6 19:10:18 CST 2024] <<<<<< START storage consumer in storage_csv_update case >>>>>> + cd /tmp/tidb_cdc_test/storage_csv_update + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/storage_csv_update + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/storage_csv_update/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml table ddl_manager.finish_mark not exists for 50-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_partition_table Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:10:20 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/75d36d1f-e030-4d39-ad66-046429bbb38b {"id":"75d36d1f-e030-4d39-ad66-046429bbb38b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993817} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98b059cd 75d36d1f-e030-4d39-ad66-046429bbb38b /tidb/cdc/default/default/upstream/7365842297033770143 {"id":7365842297033770143,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/75d36d1f-e030-4d39-ad66-046429bbb38b {"id":"75d36d1f-e030-4d39-ad66-046429bbb38b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993817} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98b059cd 75d36d1f-e030-4d39-ad66-046429bbb38b /tidb/cdc/default/default/upstream/7365842297033770143 {"id":7365842297033770143,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/75d36d1f-e030-4d39-ad66-046429bbb38b {"id":"75d36d1f-e030-4d39-ad66-046429bbb38b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993817} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98b059cd 75d36d1f-e030-4d39-ad66-046429bbb38b /tidb/cdc/default/default/upstream/7365842297033770143 {"id":7365842297033770143,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + set +x + tso='449575339605360641 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575339605360641 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:10:20 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.26512653.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + workdir=/tmp/tidb_cdc_test/kv_client_stream_reconnect + sink_uri='file:///tmp/tidb_cdc_test/kv_client_stream_reconnect/storage_test/kv-client-stream-reconnect-26502?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/kv_client_stream_reconnect ++ date + echo '[Mon May 6 19:10:20 CST 2024] <<<<<< START storage consumer in kv_client_stream_reconnect case >>>>>>' [Mon May 6 19:10:20 CST 2024] <<<<<< START storage consumer in kv_client_stream_reconnect case >>>>>> + cd /tmp/tidb_cdc_test/kv_client_stream_reconnect + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/kv_client_stream_reconnect + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/kv_client_stream_reconnect/storage_test/kv-client-stream-reconnect-26502?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' + set +x + tso='449575339737219073 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575339737219073 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:10:20 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.27412743.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449575339889262593 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575339889262593 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:10:20 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e606479a-600f-49ce-85d0-b06ee31a273e {"id":"e606479a-600f-49ce-85d0-b06ee31a273e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993818} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98a535d6 e606479a-600f-49ce-85d0-b06ee31a273e /tidb/cdc/default/default/upstream/7365842288859051908 {"id":7365842288859051908,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e606479a-600f-49ce-85d0-b06ee31a273e {"id":"e606479a-600f-49ce-85d0-b06ee31a273e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993818} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98a535d6 e606479a-600f-49ce-85d0-b06ee31a273e /tidb/cdc/default/default/upstream/7365842288859051908 {"id":7365842288859051908,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e606479a-600f-49ce-85d0-b06ee31a273e {"id":"e606479a-600f-49ce-85d0-b06ee31a273e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993818} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98a535d6 e606479a-600f-49ce-85d0-b06ee31a273e /tidb/cdc/default/default/upstream/7365842288859051908 {"id":7365842288859051908,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2706.out cli changefeed create --start-ts=449575338981982210 '--sink-uri=file:///tmp/tidb_cdc_test/foreign_key/storage_test/ticdc-foreign-key-test-30212?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: 54e0adc0-0524-4bf6-a382-22cc09420b54 Info: {"upstream_id":7365842288859051908,"namespace":"default","id":"54e0adc0-0524-4bf6-a382-22cc09420b54","sink_uri":"file:///tmp/tidb_cdc_test/foreign_key/storage_test/ticdc-foreign-key-test-30212?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:10:21.319526952+08:00","start_ts":449575338981982210,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575338981982210,"checkpoint_ts":449575338981982210,"checkpoint_time":"2024-05-06 19:10:16.307"} PASS + set +x + tso='449575339911544833 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575339911544833 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:10:21 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.27382740.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 coverage: 2.5% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... check diff failed 2-th time, retry later TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3663b32c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:1373, start at 2024-05-06 19:10:18.851014243 +0800 CST m=+5.267971804 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:18.860 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:18.827 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:18.827 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3663b5800015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:1451, start at 2024-05-06 19:10:19.022217475 +0800 CST m=+5.373617361 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:19.028 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:19.026 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:19.026 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/simple/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/simple/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/simple/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 51-th check, retry later ***************** properties ***************** "insertproportion"="0" "threadcount"="2" "workload"="core" "mysql.user"="root" "mysql.db"="sink_retry" "dotransactions"="false" "requestdistribution"="uniform" "recordcount"="10" "updateproportion"="0" "scanproportion"="0" "readallfields"="true" "mysql.port"="4000" "mysql.host"="127.0.0.1" "readproportion"="0" "operationcount"="0" ********************************************** Run finished, takes 10.009577ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1615.6, Avg(us): 1902, Min(us): 1243, Max(us): 3680, 95th(us): 4000, 99th(us): 4000 [Mon May 6 19:10:22 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.26492651.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + workdir=/tmp/tidb_cdc_test/foreign_key + sink_uri='file:///tmp/tidb_cdc_test/foreign_key/storage_test/ticdc-foreign-key-test-30212?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/foreign_key ++ date + echo '[Mon May 6 19:10:22 CST 2024] <<<<<< START storage consumer in foreign_key case >>>>>>' [Mon May 6 19:10:22 CST 2024] <<<<<< START storage consumer in foreign_key case >>>>>> + cd /tmp/tidb_cdc_test/foreign_key + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/foreign_key + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/foreign_key/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/foreign_key/storage_test/ticdc-foreign-key-test-30212?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:10:22 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4ee08b7c-63ef-4c65-b4c0-d3690d89e3b9 {"id":"4ee08b7c-63ef-4c65-b4c0-d3690d89e3b9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993820} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98a76665 4ee08b7c-63ef-4c65-b4c0-d3690d89e3b9 /tidb/cdc/default/default/upstream/7365842303354841170 {"id":7365842303354841170,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4ee08b7c-63ef-4c65-b4c0-d3690d89e3b9 {"id":"4ee08b7c-63ef-4c65-b4c0-d3690d89e3b9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993820} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98a76665 4ee08b7c-63ef-4c65-b4c0-d3690d89e3b9 /tidb/cdc/default/default/upstream/7365842303354841170 {"id":7365842303354841170,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4ee08b7c-63ef-4c65-b4c0-d3690d89e3b9 {"id":"4ee08b7c-63ef-4c65-b4c0-d3690d89e3b9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993820} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98a76665 4ee08b7c-63ef-4c65-b4c0-d3690d89e3b9 /tidb/cdc/default/default/upstream/7365842303354841170 {"id":7365842303354841170,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/processor_err_chan + sink_uri='file:///tmp/tidb_cdc_test/processor_err_chan/storage_test/ticdc-processor-err-chan-30956?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/processor_err_chan ++ date + echo '[Mon May 6 19:10:23 CST 2024] <<<<<< START storage consumer in processor_err_chan case >>>>>>' [Mon May 6 19:10:23 CST 2024] <<<<<< START storage consumer in processor_err_chan case >>>>>> + cd /tmp/tidb_cdc_test/processor_err_chan + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/processor_err_chan + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/processor_err_chan/storage_test/ticdc-processor-err-chan-30956?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' check_changefeed_state http://127.0.0.1:2379 021c8115-ff74-4622-b4fe-284cb162a55f normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=021c8115-ff74-4622-b4fe-284cb162a55f + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 021c8115-ff74-4622-b4fe-284cb162a55f -s + info='{ "upstream_id": 7365842303354841170, "namespace": "default", "id": "021c8115-ff74-4622-b4fe-284cb162a55f", "state": "normal", "checkpoint_tso": 449575340727599106, "checkpoint_time": "2024-05-06 19:10:22.966", "error": null }' + echo '{ "upstream_id": 7365842303354841170, "namespace": "default", "id": "021c8115-ff74-4622-b4fe-284cb162a55f", "state": "normal", "checkpoint_tso": 449575340727599106, "checkpoint_time": "2024-05-06 19:10:22.966", "error": null }' { "upstream_id": 7365842303354841170, "namespace": "default", "id": "021c8115-ff74-4622-b4fe-284cb162a55f", "state": "normal", "checkpoint_tso": 449575340727599106, "checkpoint_time": "2024-05-06 19:10:22.966", "error": null } ++ echo '{' '"upstream_id":' 7365842303354841170, '"namespace":' '"default",' '"id":' '"021c8115-ff74-4622-b4fe-284cb162a55f",' '"state":' '"normal",' '"checkpoint_tso":' 449575340727599106, '"checkpoint_time":' '"2024-05-06' '19:10:22.966",' '"error":' null '}' ++ jq -r .state + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:10:23 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ab4db03-dcc5-4b7d-9b70-8eb5d35d5a7b {"id":"0ab4db03-dcc5-4b7d-9b70-8eb5d35d5a7b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993820} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98af5dd5 0ab4db03-dcc5-4b7d-9b70-8eb5d35d5a7b /tidb/cdc/default/default/upstream/7365842297926490690 {"id":7365842297926490690,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ab4db03-dcc5-4b7d-9b70-8eb5d35d5a7b {"id":"0ab4db03-dcc5-4b7d-9b70-8eb5d35d5a7b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993820} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98af5dd5 0ab4db03-dcc5-4b7d-9b70-8eb5d35d5a7b /tidb/cdc/default/default/upstream/7365842297926490690 {"id":7365842297926490690,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ab4db03-dcc5-4b7d-9b70-8eb5d35d5a7b {"id":"0ab4db03-dcc5-4b7d-9b70-8eb5d35d5a7b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993820} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98af5dd5 0ab4db03-dcc5-4b7d-9b70-8eb5d35d5a7b /tidb/cdc/default/default/upstream/7365842297926490690 {"id":7365842297926490690,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2737.out cli changefeed create --start-ts=449575339545329665 '--sink-uri=file:///tmp/tidb_cdc_test/row_format/storage_test/ticdc-row-format-test-555?protocol=canal-json&enable-tidb-extension=true' + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365842303354841170, '"namespace":' '"default",' '"id":' '"021c8115-ff74-4622-b4fe-284cb162a55f",' '"state":' '"normal",' '"checkpoint_tso":' 449575340727599106, '"checkpoint_time":' '"2024-05-06' '19:10:22.966",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully Create changefeed successfully! ID: b1c329a1-5de3-4861-b6c2-2ccbded590ac Info: {"upstream_id":7365842297926490690,"namespace":"default","id":"b1c329a1-5de3-4861-b6c2-2ccbded590ac","sink_uri":"file:///tmp/tidb_cdc_test/row_format/storage_test/ticdc-row-format-test-555?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:10:23.480085384+08:00","start_ts":449575339545329665,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575339545329665,"checkpoint_ts":449575339545329665,"checkpoint_time":"2024-05-06 19:10:18.456"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check diff failed 3-th time, retry later check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:10:23 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c42e2502-16ed-441a-9ddb-7ee0b59f460b {"id":"c42e2502-16ed-441a-9ddb-7ee0b59f460b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993820} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98b12fd7 c42e2502-16ed-441a-9ddb-7ee0b59f460b /tidb/cdc/default/default/upstream/7365842303996568736 {"id":7365842303996568736,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c42e2502-16ed-441a-9ddb-7ee0b59f460b {"id":"c42e2502-16ed-441a-9ddb-7ee0b59f460b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993820} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98b12fd7 c42e2502-16ed-441a-9ddb-7ee0b59f460b /tidb/cdc/default/default/upstream/7365842303996568736 {"id":7365842303996568736,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c42e2502-16ed-441a-9ddb-7ee0b59f460b {"id":"c42e2502-16ed-441a-9ddb-7ee0b59f460b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993820} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98b12fd7 c42e2502-16ed-441a-9ddb-7ee0b59f460b /tidb/cdc/default/default/upstream/7365842303996568736 {"id":7365842303996568736,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 7dc396a8-343e-40e2-8c15-37c63bf667b0 Info: {"upstream_id":7365842303996568736,"namespace":"default","id":"7dc396a8-343e-40e2-8c15-37c63bf667b0","sink_uri":"file:///tmp/tidb_cdc_test/tiflash/storage_test/ticdc-tiflash-test-14873?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:10:23.621952294+08:00","start_ts":449575339605360641,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575339605360641,"checkpoint_ts":449575339605360641,"checkpoint_time":"2024-05-06 19:10:18.685"} + workdir=/tmp/tidb_cdc_test/tiflash + sink_uri='file:///tmp/tidb_cdc_test/tiflash/storage_test/ticdc-tiflash-test-14873?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/tiflash ++ date + echo '[Mon May 6 19:10:23 CST 2024] <<<<<< START storage consumer in tiflash case >>>>>>' [Mon May 6 19:10:23 CST 2024] <<<<<< START storage consumer in tiflash case >>>>>> + cd /tmp/tidb_cdc_test/tiflash + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/tiflash + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/tiflash/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/tiflash/storage_test/ticdc-tiflash-test-14873?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table ddl_manager.finish_mark not exists for 52-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:10:23 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a04d886-8a60-41cc-a124-d654f40c7221 {"id":"9a04d886-8a60-41cc-a124-d654f40c7221","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993821} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98b4f4ef 9a04d886-8a60-41cc-a124-d654f40c7221 /tidb/cdc/default/default/upstream/7365842306060928373 {"id":7365842306060928373,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a04d886-8a60-41cc-a124-d654f40c7221 {"id":"9a04d886-8a60-41cc-a124-d654f40c7221","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993821} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98b4f4ef 9a04d886-8a60-41cc-a124-d654f40c7221 /tidb/cdc/default/default/upstream/7365842306060928373 {"id":7365842306060928373,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a04d886-8a60-41cc-a124-d654f40c7221 {"id":"9a04d886-8a60-41cc-a124-d654f40c7221","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993821} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98b4f4ef 9a04d886-8a60-41cc-a124-d654f40c7221 /tidb/cdc/default/default/upstream/7365842306060928373 {"id":7365842306060928373,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.2705.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:10:24 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8c951866-10a0-46b4-935b-c427da9a82b5 {"id":"8c951866-10a0-46b4-935b-c427da9a82b5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993821} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98b278d0 8c951866-10a0-46b4-935b-c427da9a82b5 /tidb/cdc/default/default/upstream/7365842307661950878 {"id":7365842307661950878,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8c951866-10a0-46b4-935b-c427da9a82b5 {"id":"8c951866-10a0-46b4-935b-c427da9a82b5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993821} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98b278d0 8c951866-10a0-46b4-935b-c427da9a82b5 /tidb/cdc/default/default/upstream/7365842307661950878 {"id":7365842307661950878,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8c951866-10a0-46b4-935b-c427da9a82b5 {"id":"8c951866-10a0-46b4-935b-c427da9a82b5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993821} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98b278d0 8c951866-10a0-46b4-935b-c427da9a82b5 /tidb/cdc/default/default/upstream/7365842307661950878 {"id":7365842307661950878,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2799.out cli changefeed create --start-ts=449575339911544833 '--sink-uri=file:///tmp/tidb_cdc_test/resolve_lock/storage_test/ticdc-resolve-lock-test-25782?protocol=canal-json&enable-tidb-extension=true' table foreign_key.finish_mark not exists for 1-th check, retry later + set +x + workdir=/tmp/tidb_cdc_test/row_format + sink_uri='file:///tmp/tidb_cdc_test/row_format/storage_test/ticdc-row-format-test-555?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/row_format ++ date + echo '[Mon May 6 19:10:24 CST 2024] <<<<<< START storage consumer in row_format case >>>>>>' [Mon May 6 19:10:24 CST 2024] <<<<<< START storage consumer in row_format case >>>>>> + cd /tmp/tidb_cdc_test/row_format + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/row_format + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/row_format/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/row_format/storage_test/ticdc-row-format-test-555?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' Create changefeed successfully! ID: 5a546e2f-5cc5-4166-b82a-78ff76be7675 Info: {"upstream_id":7365842307661950878,"namespace":"default","id":"5a546e2f-5cc5-4166-b82a-78ff76be7675","sink_uri":"file:///tmp/tidb_cdc_test/resolve_lock/storage_test/ticdc-resolve-lock-test-25782?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:10:24.932436006+08:00","start_ts":449575339911544833,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575339911544833,"checkpoint_ts":449575339911544833,"checkpoint_time":"2024-05-06 19:10:19.853"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + workdir=/tmp/tidb_cdc_test/capture_session_done_during_task + sink_uri='file:///tmp/tidb_cdc_test/capture_session_done_during_task/storage_test/ticdc-capture-session-done-during-task-15144?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/capture_session_done_during_task ++ date + echo '[Mon May 6 19:10:24 CST 2024] <<<<<< START storage consumer in capture_session_done_during_task case >>>>>>' [Mon May 6 19:10:24 CST 2024] <<<<<< START storage consumer in capture_session_done_during_task case >>>>>> + cd /tmp/tidb_cdc_test/capture_session_done_during_task + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/capture_session_done_during_task + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/capture_session_done_during_task/storage_test/ticdc-capture-session-done-during-task-15144?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' lease 22318f4d98b4f4ef revoked + set +x + tso='449575341058424833 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575341058424833 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table ddl_manager.finish_mark not exists for 53-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:10:25 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a95a003-8a2f-4866-b259-9061995a8c16 {"id":"7a95a003-8a2f-4866-b259-9061995a8c16","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993822} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98ae09f7 7a95a003-8a2f-4866-b259-9061995a8c16 /tidb/cdc/default/default/upstream/7365842300388913977 {"id":7365842300388913977,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a95a003-8a2f-4866-b259-9061995a8c16 {"id":"7a95a003-8a2f-4866-b259-9061995a8c16","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993822} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98ae09f7 7a95a003-8a2f-4866-b259-9061995a8c16 /tidb/cdc/default/default/upstream/7365842300388913977 {"id":7365842300388913977,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a95a003-8a2f-4866-b259-9061995a8c16 {"id":"7a95a003-8a2f-4866-b259-9061995a8c16","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993822} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98ae09f7 7a95a003-8a2f-4866-b259-9061995a8c16 /tidb/cdc/default/default/upstream/7365842300388913977 {"id":7365842300388913977,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2704.out cli changefeed create --start-ts=449575339889262593 '--sink-uri=file:///tmp/tidb_cdc_test/sink_retry/storage_test/ticdc-sink-retry-test-31886?protocol=canal-json&enable-tidb-extension=true' [Mon May 6 19:10:25 CST 2024] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.27612763.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 4-th time, retry later Create changefeed successfully! ID: 6b7ae6d1-a88f-471e-bfc7-4e5fa5843596 Info: {"upstream_id":7365842300388913977,"namespace":"default","id":"6b7ae6d1-a88f-471e-bfc7-4e5fa5843596","sink_uri":"file:///tmp/tidb_cdc_test/sink_retry/storage_test/ticdc-sink-retry-test-31886?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:10:26.003294108+08:00","start_ts":449575339889262593,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575339889262593,"checkpoint_ts":449575339889262593,"checkpoint_time":"2024-05-06 19:10:19.768"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table capture_session_done_during_task.t exists check diff failed 1-th time, retry later table cdc_tiflash_test.multi_data_type not exists for 2-th check, retry later check diff failed 1-th time, retry later + set +x + workdir=/tmp/tidb_cdc_test/resolve_lock + sink_uri='file:///tmp/tidb_cdc_test/resolve_lock/storage_test/ticdc-resolve-lock-test-25782?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/resolve_lock ++ date + echo '[Mon May 6 19:10:26 CST 2024] <<<<<< START storage consumer in resolve_lock case >>>>>>' [Mon May 6 19:10:26 CST 2024] <<<<<< START storage consumer in resolve_lock case >>>>>> + cd /tmp/tidb_cdc_test/resolve_lock + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/resolve_lock + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/resolve_lock/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/resolve_lock/storage_test/ticdc-resolve-lock-test-25782?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/google/uuid v1.6.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading golang.org/x/sync v0.7.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/sys v0.19.0 table foreign_key.finish_mark not exists for 2-th check, retry later go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda table ddl_manager.finish_mark not exists for 54-th check, retry later go: downloading golang.org/x/text v0.14.0 check diff failed 5-th time, retry later + set +x + workdir=/tmp/tidb_cdc_test/sink_retry + sink_uri='file:///tmp/tidb_cdc_test/sink_retry/storage_test/ticdc-sink-retry-test-31886?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/sink_retry ++ date + echo '[Mon May 6 19:10:27 CST 2024] <<<<<< START storage consumer in sink_retry case >>>>>>' [Mon May 6 19:10:27 CST 2024] <<<<<< START storage consumer in sink_retry case >>>>>> + cd /tmp/tidb_cdc_test/sink_retry + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/sink_retry + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/sink_retry/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/sink_retry/storage_test/ticdc-sink-retry-test-31886?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later table cdc_tiflash_test.multi_data_type not exists for 3-th check, retry later check diff failed 2-th time, retry later table foreign_key.finish_mark not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:10:29 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8a9c1fa4-55a3-41a8-97ff-e3e15d1660a8 {"id":"8a9c1fa4-55a3-41a8-97ff-e3e15d1660a8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993826} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98c06fef 8a9c1fa4-55a3-41a8-97ff-e3e15d1660a8 /tidb/cdc/default/default/upstream/7365842329819772881 {"id":7365842329819772881,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8a9c1fa4-55a3-41a8-97ff-e3e15d1660a8 {"id":"8a9c1fa4-55a3-41a8-97ff-e3e15d1660a8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993826} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98c06fef 8a9c1fa4-55a3-41a8-97ff-e3e15d1660a8 /tidb/cdc/default/default/upstream/7365842329819772881 {"id":7365842329819772881,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8a9c1fa4-55a3-41a8-97ff-e3e15d1660a8 {"id":"8a9c1fa4-55a3-41a8-97ff-e3e15d1660a8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993826} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98c06fef 8a9c1fa4-55a3-41a8-97ff-e3e15d1660a8 /tidb/cdc/default/default/upstream/7365842329819772881 {"id":7365842329819772881,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.2824.out cli changefeed create --start-ts=449575341058424833 '--sink-uri=file:///tmp/tidb_cdc_test/simple/storage_test/ticdc-simple-test-19405?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: e9086ccc-7fc9-4766-b4b1-19fd830a9bac Info: {"upstream_id":7365842329819772881,"namespace":"default","id":"e9086ccc-7fc9-4766-b4b1-19fd830a9bac","sink_uri":"file:///tmp/tidb_cdc_test/simple/storage_test/ticdc-simple-test-19405?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:10:29.436644929+08:00","start_ts":449575341058424833,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575341058424833,"checkpoint_ts":449575341058424833,"checkpoint_time":"2024-05-06 19:10:24.228"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading google.golang.org/api v0.170.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 check diff failed 6-th time, retry later go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/kr/pretty v0.3.1 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/lestrrat-go/httpcc v1.0.1 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3664556c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55, pid:9269, start at 2024-05-06 19:10:29.243095916 +0800 CST m=+5.437319391 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:29.252 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:29.211 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:29.211 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible table ddl_manager.finish_mark not exists for 55-th check, retry later table cdc_tiflash_test.multi_data_type not exists for 4-th check, retry later go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da check diff failed 4-th time, retry later check diff failed 3-th time, retry later go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 + set +x + workdir=/tmp/tidb_cdc_test/simple + sink_uri='file:///tmp/tidb_cdc_test/simple/storage_test/ticdc-simple-test-19405?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/simple ++ date + echo '[Mon May 6 19:10:30 CST 2024] <<<<<< START storage consumer in simple case >>>>>>' [Mon May 6 19:10:30 CST 2024] <<<<<< START storage consumer in simple case >>>>>> + cd /tmp/tidb_cdc_test/simple + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/simple + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/simple/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/simple/storage_test/ticdc-simple-test-19405?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' succeed to verify meta placement rules go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 check diff successfully [Mon May 6 19:10:30 CST 2024] <<<<<< run test case storage_csv_update success! >>>>>> ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 1-th time, retry later table foreign_key.finish_mark not exists for 4-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... go: downloading k8s.io/apimachinery v0.28.6 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3664556c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55, pid:9269, start at 2024-05-06 19:10:29.243095916 +0800 CST m=+5.437319391 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:29.252 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:29.211 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:29.211 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366465080013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6cp-lzr55, pid:9359, start at 2024-05-06 19:10:30.243780166 +0800 CST m=+6.382216486 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:30.253 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:30.260 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:30.260 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_storage_partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_storage_partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_storage_partition_table/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_storage_partition_table/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_storage_partition_table/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 56-th check, retry later table cdc_tiflash_test.multi_data_type not exists for 5-th check, retry later check diff failed 7-th time, retry later check diff failed 5-th time, retry later check diff failed 4-th time, retry later table foreign_key.finish_mark not exists for 5-th check, retry later ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table cdc_tiflash_test.multi_data_type exists table ddl_manager.finish_mark not exists for 57-th check, retry later check diff failed 8-th time, retry later check diff successfully check diff failed 6-th time, retry later check diff failed 5-th time, retry later [Mon May 6 19:10:34 CST 2024] <<<<<< START cdc server in canal_json_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.1062810630.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdccdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_datacdc0 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 wait process cdc.test exit for 1-th time... table foreign_key.finish_mark not exists for 6-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 19.54 secs (190924481 bytes/sec) [Pipeline] { [Pipeline] cache wait process cdc.test exit for 2-th time... ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 3-th time, retry later table ddl_manager.finish_mark not exists for 58-th check, retry later wait process cdc.test exit for 3-th time... check diff successfully cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:10:36 CST 2024] <<<<<< run test case tiflash success! >>>>>> check diff successfully check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:10:37 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1be656be-e90a-4058-b01b-f3fd8ed07d18 {"id":"1be656be-e90a-4058-b01b-f3fd8ed07d18","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993834} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98f03acf 1be656be-e90a-4058-b01b-f3fd8ed07d18 /tidb/cdc/default/default/upstream/7365842371895597113 {"id":7365842371895597113,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1be656be-e90a-4058-b01b-f3fd8ed07d18 {"id":"1be656be-e90a-4058-b01b-f3fd8ed07d18","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993834} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98f03acf 1be656be-e90a-4058-b01b-f3fd8ed07d18 /tidb/cdc/default/default/upstream/7365842371895597113 {"id":7365842371895597113,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1be656be-e90a-4058-b01b-f3fd8ed07d18 {"id":"1be656be-e90a-4058-b01b-f3fd8ed07d18","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993834} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98f03acf 1be656be-e90a-4058-b01b-f3fd8ed07d18 /tidb/cdc/default/default/upstream/7365842371895597113 {"id":7365842371895597113,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:10:37 CST 2024] <<<<<< START cdc server in canal_json_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 4-th time, retry later + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.1068810690.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdccdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_datacdc1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table foreign_key.finish_mark not exists for 7-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 59-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 2-th time... check diff failed 2-th time, retry later ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 5-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... table foreign_key.finish_mark not exists for 8-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:10:38 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 60-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:10:40 GMT < Content-Length: 1275 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0fc675ca-45ce-45e5-b597-75d1a25039b7 {"id":"0fc675ca-45ce-45e5-b597-75d1a25039b7","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993837} /tidb/cdc/default/__cdc_meta__/capture/1be656be-e90a-4058-b01b-f3fd8ed07d18 {"id":"1be656be-e90a-4058-b01b-f3fd8ed07d18","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993834} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98f03acf 1be656be-e90a-4058-b01b-f3fd8ed07d18 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98f03b01 0fc675ca-45ce-45e5-b597-75d1a25039b7 /tidb/cdc/default/default/upstream/7365842371895597113 {"id":7365842371895597113,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0fc675ca-45ce-45e5-b597-75d1a25039b7 {"id":"0fc675ca-45ce-45e5-b597-75d1a25039b7","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993837} /tidb/cdc/default/__cdc_meta__/capture/1be656be-e90a-4058-b01b-f3fd8ed07d18 {"id":"1be656be-e90a-4058-b01b-f3fd8ed07d18","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993834} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98f03acf 1be656be-e90a-4058-b01b-f3fd8ed07d18 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98f03b01 0fc675ca-45ce-45e5-b597-75d1a25039b7 /tidb/cdc/default/default/upstream/7365842371895597113 {"id":7365842371895597113,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0fc675ca-45ce-45e5-b597-75d1a25039b7 {"id":"0fc675ca-45ce-45e5-b597-75d1a25039b7","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993837} /tidb/cdc/default/__cdc_meta__/capture/1be656be-e90a-4058-b01b-f3fd8ed07d18 {"id":"1be656be-e90a-4058-b01b-f3fd8ed07d18","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993834} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98f03acf 1be656be-e90a-4058-b01b-f3fd8ed07d18 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98f03b01 0fc675ca-45ce-45e5-b597-75d1a25039b7 /tidb/cdc/default/default/upstream/7365842371895597113 {"id":7365842371895597113,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:10:40 CST 2024] <<<<<< START cdc server in canal_json_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.1074110743.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdccdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_datacdc2 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table foreign_key.finish_mark not exists for 9-th check, retry later check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 1146 (42S02) at line 1: Table 'test.simple2' doesn't exist check data failed 6-th time, retry later table ddl_manager.finish_mark not exists for 61-th check, retry later check diff failed 2-th time, retry later table row_format.finish_mark not exists for 1-th check, retry later table row_format.finish_mark not exists for 2-th check, retry later table foreign_key.finish_mark not exists for 10-th check, retry later check diff failed 4-th time, retry later check data successfully wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:10:43 GMT < Content-Length: 1756 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0fc675ca-45ce-45e5-b597-75d1a25039b7 {"id":"0fc675ca-45ce-45e5-b597-75d1a25039b7","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993837} /tidb/cdc/default/__cdc_meta__/capture/1be656be-e90a-4058-b01b-f3fd8ed07d18 {"id":"1be656be-e90a-4058-b01b-f3fd8ed07d18","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993834} /tidb/cdc/default/__cdc_meta__/capture/afcd5830-0493-447c-9781-e91f42d8f86d {"id":"afcd5830-0493-447c-9781-e91f42d8f86d","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993840} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98f03acf 1be656be-e90a-4058-b01b-f3fd8ed07d18 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98f03b01 0fc675ca-45ce-45e5-b597-75d1a25039b7 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98f03b1c afcd5830-0493-447c-9781-e91f42d8f86d /tidb/cdc/default/default/upstream/7365842371895597113 {"id":7365842371895597113,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0fc675ca-45ce-45e5-b597-75d1a25039b7 {"id":"0fc675ca-45ce-45e5-b597-75d1a25039b7","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993837} /tidb/cdc/default/__cdc_meta__/capture/1be656be-e90a-4058-b01b-f3fd8ed07d18 {"id":"1be656be-e90a-4058-b01b-f3fd8ed07d18","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993834} /tidb/cdc/default/__cdc_meta__/capture/afcd5830-0493-447c-9781-e91f42d8f86d {"id":"afcd5830-0493-447c-9781-e91f42d8f86d","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993840} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98f03acf 1be656be-e90a-4058-b01b-f3fd8ed07d18 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98f03b01 0fc675ca-45ce-45e5-b597-75d1a25039b7 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98f03b1c afcd5830-0493-447c-9781-e91f42d8f86d /tidb/cdc/default/default/upstream/7365842371895597113 {"id":7365842371895597113,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0fc675ca-45ce-45e5-b597-75d1a25039b7 {"id":"0fc675ca-45ce-45e5-b597-75d1a25039b7","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993837} /tidb/cdc/default/__cdc_meta__/capture/1be656be-e90a-4058-b01b-f3fd8ed07d18 {"id":"1be656be-e90a-4058-b01b-f3fd8ed07d18","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993834} /tidb/cdc/default/__cdc_meta__/capture/afcd5830-0493-447c-9781-e91f42d8f86d {"id":"afcd5830-0493-447c-9781-e91f42d8f86d","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993840} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98f03acf 1be656be-e90a-4058-b01b-f3fd8ed07d18 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98f03b01 0fc675ca-45ce-45e5-b597-75d1a25039b7 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98f03b1c afcd5830-0493-447c-9781-e91f42d8f86d /tidb/cdc/default/default/upstream/7365842371895597113 {"id":7365842371895597113,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.cli.10796.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml Create changefeed successfully! ID: 6a50f0f6-fd7b-468c-8308-5e9be6c22378 Info: {"upstream_id":7365842371895597113,"namespace":"default","id":"6a50f0f6-fd7b-468c-8308-5e9be6c22378","sink_uri":"file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:10:43.84215841+08:00","start_ts":449575346165514250,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575346165514250,"checkpoint_ts":449575346165514250,"checkpoint_time":"2024-05-06 19:10:43.710"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 62-th check, retry later wait process cdc.test exit for 2-th time... check diff failed 3-th time, retry later table row_format.finish_mark not exists for 3-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:10:44 CST 2024] <<<<<< run test case simple success! >>>>>> table foreign_key.finish_mark not exists for 11-th check, retry later check diff failed 5-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 7.38 secs (505229860 bytes/sec) [Pipeline] { [Pipeline] cache + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m table ddl_manager.finish_mark not exists for 63-th check, retry later check diff failed 4-th time, retry later table row_format.finish_mark not exists for 4-th check, retry later table foreign_key.finish_mark not exists for 12-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 64-th check, retry later wait process cdc.test exit for 2-th time... check diff failed 5-th time, retry later table row_format.finish_mark not exists for 5-th check, retry later + workdir=/tmp/tidb_cdc_test/canal_json_storage_partition_table + sink_uri='file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/canal_json_storage_partition_table ++ date + echo '[Mon May 6 19:10:48 CST 2024] <<<<<< START storage consumer in canal_json_storage_partition_table case >>>>>>' [Mon May 6 19:10:48 CST 2024] <<<<<< START storage consumer in canal_json_storage_partition_table case >>>>>> + cd /tmp/tidb_cdc_test/canal_json_storage_partition_table + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/canal_json_storage_partition_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:10:48 CST 2024] <<<<<< run test case capture_session_done_during_task success! >>>>>> table foreign_key.finish_mark not exists for 13-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366572140016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9, pid:7556, start at 2024-05-06 19:10:47.460766231 +0800 CST m=+8.088257619 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:47.473 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:47.478 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:47.478 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366572140016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9, pid:7556, start at 2024-05-06 19:10:47.460766231 +0800 CST m=+8.088257619 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:47.473 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:47.478 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:47.478 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366573c00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9, pid:7627, start at 2024-05-06 19:10:47.57741742 +0800 CST m=+8.127085218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:47.588 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:47.587 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:47.587 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 65-th check, retry later table row_format.finish_mark not exists for 6-th check, retry later check diff failed 6-th time, retry later table foreign_key.finish_mark not exists for 14-th check, retry later [Mon May 6 19:10:51 CST 2024] <<<<<< START cdc server in csv_storage_multi_tables_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + GO_FAILPOINTS= + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_multi_tables_ddl.88968898.out server --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 66-th check, retry later check diff failed 7-th time, retry later table row_format.finish_mark not exists for 7-th check, retry later table foreign_key.finish_mark not exists for 15-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 67-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 7.17 secs (519980573 bytes/sec) [Pipeline] { check diff successfully [Pipeline] cache table row_format.finish_mark not exists for 8-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:10:55 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74ea5794-3908-4303-9dcf-271f72fd9382 {"id":"74ea5794-3908-4303-9dcf-271f72fd9382","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993852} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99272cd4 74ea5794-3908-4303-9dcf-271f72fd9382 /tidb/cdc/default/default/upstream/7365842438693335064 {"id":7365842438693335064,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74ea5794-3908-4303-9dcf-271f72fd9382 {"id":"74ea5794-3908-4303-9dcf-271f72fd9382","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993852} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99272cd4 74ea5794-3908-4303-9dcf-271f72fd9382 /tidb/cdc/default/default/upstream/7365842438693335064 {"id":7365842438693335064,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74ea5794-3908-4303-9dcf-271f72fd9382 {"id":"74ea5794-3908-4303-9dcf-271f72fd9382","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993852} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99272cd4 74ea5794-3908-4303-9dcf-271f72fd9382 /tidb/cdc/default/default/upstream/7365842438693335064 {"id":7365842438693335064,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7365842438693335064,"namespace":"default","id":"test-normal","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-normal-19414?flush-interval=5s\u0026protocol=csv","create_time":"2024-05-06T19:10:55.194550843+08:00","start_ts":449575348306706433,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575348306706433,"checkpoint_ts":449575348306706433,"checkpoint_time":"2024-05-06 19:10:51.878"} table foreign_key.finish_mark not exists for 16-th check, retry later Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7365842438693335064,"namespace":"default","id":"test-error-1","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-1-21436?flush-interval=5s\u0026protocol=csv","create_time":"2024-05-06T19:10:55.393395102+08:00","start_ts":449575348306706433,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575348306706433,"checkpoint_ts":449575348306706433,"checkpoint_time":"2024-05-06 19:10:51.878"} Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7365842438693335064,"namespace":"default","id":"test-error-2","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-2-31545?flush-interval=5s\u0026protocol=csv","create_time":"2024-05-06T19:10:55.54296029+08:00","start_ts":449575348306706433,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575348306706433,"checkpoint_ts":449575348306706433,"checkpoint_time":"2024-05-06 19:10:51.878"} + workdir=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-normal-19414?flush-interval=5s&protocol=csv' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml + log_suffix=1 ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl ++ date + echo '[Mon May 6 19:10:55 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>>' [Mon May 6 19:10:55 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_storage_consumer1.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-normal-19414?flush-interval=5s&protocol=csv' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml + workdir=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-1-21436?flush-interval=5s&protocol=csv' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-1.toml + log_suffix=2 ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl ++ date + echo '[Mon May 6 19:10:55 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>>' [Mon May 6 19:10:55 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-1.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_storage_consumer2.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-1-21436?flush-interval=5s&protocol=csv' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-1.toml + workdir=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-2-31545?flush-interval=5s&protocol=csv' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-2.toml + log_suffix=3 ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl ++ date + echo '[Mon May 6 19:10:55 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>>' [Mon May 6 19:10:55 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-2.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_storage_consumer3.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-2-31545?flush-interval=5s&protocol=csv' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-2.toml table ddl_manager.finish_mark not exists for 68-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table partition_table.t not exists for 1-th check, retry later table row_format.finish_mark not exists for 9-th check, retry later table foreign_key.finish_mark not exists for 17-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 69-th check, retry later table partition_table.t not exists for 2-th check, retry later table row_format.finish_mark not exists for 10-th check, retry later check diff failed 1-th time, retry later table multi_tables_ddl_test.t55 not exists for 1-th check, retry later Verifying downstream PD is started... check diff failed 1-th time, retry later check diff failed 2-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... table foreign_key.finish_mark not exists for 18-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 70-th check, retry later table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "readallfields"="true" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "recordcount"="10" "updateproportion"="0" "threadcount"="2" "insertproportion"="0" "dotransactions"="false" "operationcount"="0" "mysql.port"="4000" "workload"="core" "scanproportion"="0" "readproportion"="0" "mysql.db"="sink_retry" "mysql.user"="root" ********************************************** Run finished, takes 11.859802ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2810.3, Avg(us): 2285, Min(us): 594, Max(us): 8222, 95th(us): 9000, 99th(us): 9000 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table row_format.finish_mark not exists for 11-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table partition_table.t exists table partition_table.t1 exists table partition_table.finish_mark not exists for 1-th check, retry later check diff failed 2-th time, retry later table multi_tables_ddl_test.t55 not exists for 2-th check, retry later table foreign_key.finish_mark not exists for 19-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 3-th time, retry later table row_format.finish_mark not exists for 12-th check, retry later table ddl_manager.finish_mark not exists for 71-th check, retry later table multi_tables_ddl_test.t55 not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 2-th check, retry later check diff failed 3-th time, retry later table foreign_key.finish_mark not exists for 20-th check, retry later check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 72-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 13-th check, retry later table partition_table.finish_mark not exists for 3-th check, retry later check diff failed 4-th time, retry later table multi_tables_ddl_test.t55 not exists for 4-th check, retry later table foreign_key.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 73-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 3-th time... table row_format.finish_mark not exists for 14-th check, retry later table partition_table.finish_mark not exists for 4-th check, retry later table multi_tables_ddl_test.t55 not exists for 5-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:11:07 CST 2024] <<<<<< run test case foreign_key success! >>>>>> check diff failed 5-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 6-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 74-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3666b6200017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-xp8-cvcxs, pid:3820, start at 2024-05-06 19:11:08.217115555 +0800 CST m=+5.702089567 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:08.224 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:08.218 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:08.218 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 5-th check, retry later table row_format.finish_mark not exists for 15-th check, retry later table multi_tables_ddl_test.t55 not exists for 6-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 6-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3666aa640007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:4103, start at 2024-05-06 19:11:07.419646934 +0800 CST m=+5.852620500 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:07.428 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:07.417 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:07.417 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3666aa640007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:4103, start at 2024-05-06 19:11:07.419646934 +0800 CST m=+5.852620500 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:07.428 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:07.417 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:07.417 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3666c8280011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:4167, start at 2024-05-06 19:11:09.338029091 +0800 CST m=+7.698992056 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:09.345 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:09.322 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:09.322 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 7-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3666c5840020 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:4001, start at 2024-05-06 19:11:09.202127304 +0800 CST m=+5.238902870 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:09.210 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:09.203 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:09.203 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3666c5840020 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:4001, start at 2024-05-06 19:11:09.202127304 +0800 CST m=+5.238902870 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:09.210 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:09.203 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:09.203 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3666c7000014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:4085, start at 2024-05-06 19:11:09.280263236 +0800 CST m=+5.243069049 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:09.290 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:09.248 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:09.248 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table ddl_manager.finish_mark not exists for 75-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3666b6200017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-xp8-cvcxs, pid:3820, start at 2024-05-06 19:11:08.217115555 +0800 CST m=+5.702089567 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:08.224 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:08.218 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:08.218 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3666b6440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-xp8-cvcxs, pid:3905, start at 2024-05-06 19:11:08.206650577 +0800 CST m=+5.636166406 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:08.213 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:08.177 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:08.177 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table partition_table.finish_mark not exists for 6-th check, retry later table row_format.finish_mark not exists for 16-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table multi_tables_ddl_test.t55 not exists for 7-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc) 3730550784 bytes in 13.64 secs (273437363 bytes/sec) [Pipeline] { [Pipeline] sh check diff failed 7-th time, retry later [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G14 [Pipeline] sh table ddl_manager.finish_mark not exists for 76-th check, retry later Run cases: changefeed_finish force_replicate_table PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=530df9ca-213b-4c50-9876-0df90b38b9c0 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-722-gnh4g HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-722k0 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1816-722k0 pingcap-tiflow-pull-cdc-integration-storage-test-1816-722-gnh4g GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-722-gnh4g GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.5400.out cli tso query --pd=http://127.0.0.1:2379 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G15 Run cases: new_ci_collation batch_add_table multi_rocks PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=22476a94-db79-4aca-9278-ee2d68c827ef BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-4h8-1vqcr HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-4h8pq GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1816-4h8pq pingcap-tiflow-pull-cdc-integration-storage-test-1816-4h8-1vqcr GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-4h8-1vqcr GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:11:12 CST 2024] <<<<<< run test case new_ci_collation success! >>>>>> [Pipeline] sh [Mon May 6 19:11:11 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.55415543.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table row_format.finish_mark not exists for 17-th check, retry later check diff successfully + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G16 Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=37808436-6bfd-4be9-95a8-b793f7bd10c5 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35j HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-6jw4w GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35j pingcap_tiflow_pull_cdc_integration_storage_test_1816-6jw4w GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35j GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: storage... <<================= wait process cdc.test exit for 1-th time... The 1 times to try to start tidb cluster... table multi_tables_ddl_test.t55 not exists for 8-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G17 Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=9cf00770-43ac-4456-a2f8-78f2d0994ab5 JENKINS_NODE_COOKIE=4279ac2c-b73c-444b-a723-08e7183eec8a BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-gxw-8pkgb HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1787435357051031552","prowjobid":"9cf00770-43ac-4456-a2f8-78f2d0994ab5","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1816 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357051031552 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1816/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1816-gxwrn GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1816-gxwrn pingcap-tiflow-pull-cdc-integration-storage-test-1816-gxw-8pkgb GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1816-gxw-8pkgb GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1816 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:11:12 CST 2024] <<<<<< skip test case clustered_index for storage! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 7-th check, retry later [Mon May 6 19:11:12 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.52795281.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 8-th time, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... + set +x + tso='449575353739640833 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575353739640833 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x try a VALID cdc server command [Mon May 6 19:11:14 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.54475449.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:11:14 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> table ddl_manager.finish_mark not exists for 77-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366712e8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-j5h-9wmdc, pid:4208, start at 2024-05-06 19:11:14.121692699 +0800 CST m=+5.373300394 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:14.130 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:14.106 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:14.106 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 8-th check, retry later table row_format.finish_mark not exists for 18-th check, retry later table multi_tables_ddl_test.t55 not exists for 9-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:15 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e4217ada-42ed-4ffc-8837-c947e016c915 {"id":"e4217ada-42ed-4ffc-8837-c947e016c915","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993872} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9983b5cb e4217ada-42ed-4ffc-8837-c947e016c915 /tidb/cdc/default/default/upstream/7365842536789399572 {"id":7365842536789399572,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e4217ada-42ed-4ffc-8837-c947e016c915 {"id":"e4217ada-42ed-4ffc-8837-c947e016c915","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993872} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9983b5cb e4217ada-42ed-4ffc-8837-c947e016c915 /tidb/cdc/default/default/upstream/7365842536789399572 {"id":7365842536789399572,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e4217ada-42ed-4ffc-8837-c947e016c915 {"id":"e4217ada-42ed-4ffc-8837-c947e016c915","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993872} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9983b5cb e4217ada-42ed-4ffc-8837-c947e016c915 /tidb/cdc/default/default/upstream/7365842536789399572 {"id":7365842536789399572,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/changefeed_reconstruct + sink_uri='file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-25417?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/changefeed_reconstruct ++ date + echo '[Mon May 6 19:11:15 CST 2024] <<<<<< START storage consumer in changefeed_reconstruct case >>>>>>' [Mon May 6 19:11:15 CST 2024] <<<<<< START storage consumer in changefeed_reconstruct case >>>>>> + cd /tmp/tidb_cdc_test/changefeed_reconstruct + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/changefeed_reconstruct + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-25417?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > check diff successfully < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:15 GMT < Content-Length: 861 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43323ea8-142b-41c1-a00d-4f8617987d46 {"id":"43323ea8-142b-41c1-a00d-4f8617987d46","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993873} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9983513b 43323ea8-142b-41c1-a00d-4f8617987d46 /tidb/cdc/default/default/upstream/7365842531970890363 {"id":7365842531970890363,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43323ea8-142b-41c1-a00d-4f8617987d46 {"id":"43323ea8-142b-41c1-a00d-4f8617987d46","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993873} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9983513b 43323ea8-142b-41c1-a00d-4f8617987d46 /tidb/cdc/default/default/upstream/7365842531970890363 {"id":7365842531970890363,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43323ea8-142b-41c1-a00d-4f8617987d46 {"id":"43323ea8-142b-41c1-a00d-4f8617987d46","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993873} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9983513b 43323ea8-142b-41c1-a00d-4f8617987d46 /tidb/cdc/default/default/upstream/7365842531970890363 {"id":7365842531970890363,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: eeab3387-e354-4144-baf4-7be3d12fc71f Info: {"upstream_id":7365842531970890363,"namespace":"default","id":"eeab3387-e354-4144-baf4-7be3d12fc71f","sink_uri":"file:///tmp/tidb_cdc_test/region_merge/storage_test/ticdc-region-merge-test-30360?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:11:16.143713376+08:00","start_ts":449575354635124740,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575354635124740,"checkpoint_ts":449575354635124740,"checkpoint_time":"2024-05-06 19:11:16.019"} + workdir=/tmp/tidb_cdc_test/region_merge + sink_uri='file:///tmp/tidb_cdc_test/region_merge/storage_test/ticdc-region-merge-test-30360?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/region_merge ++ date + echo '[Mon May 6 19:11:16 CST 2024] <<<<<< START storage consumer in region_merge case >>>>>>' [Mon May 6 19:11:16 CST 2024] <<<<<< START storage consumer in region_merge case >>>>>> + cd /tmp/tidb_cdc_test/region_merge + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/region_merge + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/region_merge/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/region_merge/storage_test/ticdc-region-merge-test-30360?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' split_and_random_merge scale: 20 start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 78-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... ***************** properties ***************** "scanproportion"="0" "workload"="core" "threadcount"="4" "recordcount"="50" "operationcount"="0" "insertproportion"="0" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "mysql.db"="changefeed_reconstruct" "updateproportion"="0" "mysql.port"="4000" "dotransactions"="false" "readproportion"="0" "mysql.user"="root" "readallfields"="true" ********************************************** Run finished, takes 33.230126ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3446.8, Avg(us): 2688, Min(us): 889, Max(us): 19291, 95th(us): 20000, 99th(us): 20000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366712e8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-j5h-9wmdc, pid:4208, start at 2024-05-06 19:11:14.121692699 +0800 CST m=+5.373300394 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:14.130 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:14.106 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:14.106 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366713e40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-j5h-9wmdc, pid:4281, start at 2024-05-06 19:11:14.207035299 +0800 CST m=+5.407332001 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:14.215 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:14.219 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:14.219 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table partition_table.finish_mark not exists for 9-th check, retry later table row_format.finish_mark not exists for 19-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:17 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25dbbe66-3844-48fa-8c9f-e918a7e431e3 {"id":"25dbbe66-3844-48fa-8c9f-e918a7e431e3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993874} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d998cd5dd 25dbbe66-3844-48fa-8c9f-e918a7e431e3 /tidb/cdc/default/default/upstream/7365842542853595107 {"id":7365842542853595107,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25dbbe66-3844-48fa-8c9f-e918a7e431e3 {"id":"25dbbe66-3844-48fa-8c9f-e918a7e431e3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993874} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d998cd5dd 25dbbe66-3844-48fa-8c9f-e918a7e431e3 /tidb/cdc/default/default/upstream/7365842542853595107 {"id":7365842542853595107,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25dbbe66-3844-48fa-8c9f-e918a7e431e3 {"id":"25dbbe66-3844-48fa-8c9f-e918a7e431e3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993874} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d998cd5dd 25dbbe66-3844-48fa-8c9f-e918a7e431e3 /tidb/cdc/default/default/upstream/7365842542853595107 {"id":7365842542853595107,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... table ddl_manager.finish_mark not exists for 79-th check, retry later table changefeed_reconstruct.usertable not exists for 2-th check, retry later table partition_table.finish_mark not exists for 10-th check, retry later table row_format.finish_mark not exists for 20-th check, retry later table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.5595.out cli tso query --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore check diff failed 1-th time, retry later Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 80-th check, retry later [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc already exists) table partition_table.finish_mark not exists for 11-th check, retry later table row_format.finish_mark not exists for 21-th check, retry later table changefeed_reconstruct.usertable not exists for 3-th check, retry later + set +x + tso='449575355523006465 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575355523006465 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:11:20 CST 2024] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.56265628.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table multi_tables_ddl_test.finish_mark not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 81-th check, retry later table row_format.finish_mark not exists for 22-th check, retry later table changefeed_reconstruct.usertable not exists for 4-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... *************************** 1. row *************************** count(distinct region_id): 1 table multi_tables_ddl_test.finish_mark not exists for 4-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table partition_table.finish_mark not exists for 12-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:23 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/419cac21-9a65-4c17-95d2-7f7b6e3e2ccb {"id":"419cac21-9a65-4c17-95d2-7f7b6e3e2ccb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993881} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d999dd8d6 419cac21-9a65-4c17-95d2-7f7b6e3e2ccb /tidb/cdc/default/default/upstream/7365842564766990589 {"id":7365842564766990589,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/419cac21-9a65-4c17-95d2-7f7b6e3e2ccb {"id":"419cac21-9a65-4c17-95d2-7f7b6e3e2ccb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993881} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d999dd8d6 419cac21-9a65-4c17-95d2-7f7b6e3e2ccb /tidb/cdc/default/default/upstream/7365842564766990589 {"id":7365842564766990589,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/419cac21-9a65-4c17-95d2-7f7b6e3e2ccb {"id":"419cac21-9a65-4c17-95d2-7f7b6e3e2ccb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993881} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d999dd8d6 419cac21-9a65-4c17-95d2-7f7b6e3e2ccb /tidb/cdc/default/default/upstream/7365842564766990589 {"id":7365842564766990589,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.5685.out cli changefeed create --start-ts=449575355523006465 '--sink-uri=file:///tmp/tidb_cdc_test/ddl_attributes/storage_test/ticdc-ddl-attributes-test-10687?protocol=canal-json&enable-tidb-extension=true' Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Create changefeed successfully! ID: d35d4a19-e7ff-453b-b09f-8aa8bf35a657 Info: {"upstream_id":7365842564766990589,"namespace":"default","id":"d35d4a19-e7ff-453b-b09f-8aa8bf35a657","sink_uri":"file:///tmp/tidb_cdc_test/ddl_attributes/storage_test/ticdc-ddl-attributes-test-10687?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:11:24.361091149+08:00","start_ts":449575355523006465,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575355523006465,"checkpoint_ts":449575355523006465,"checkpoint_time":"2024-05-06 19:11:19.406"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 82-th check, retry later Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_reconstruct.usertable not exists for 5-th check, retry later table partition_table.finish_mark not exists for 13-th check, retry later table row_format.finish_mark not exists for 23-th check, retry later table multi_tables_ddl_test.finish_mark not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + workdir=/tmp/tidb_cdc_test/ddl_attributes + sink_uri='file:///tmp/tidb_cdc_test/ddl_attributes/storage_test/ticdc-ddl-attributes-test-10687?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/ddl_attributes ++ date + echo '[Mon May 6 19:11:25 CST 2024] <<<<<< START storage consumer in ddl_attributes case >>>>>>' [Mon May 6 19:11:25 CST 2024] <<<<<< START storage consumer in ddl_attributes case >>>>>> + cd /tmp/tidb_cdc_test/ddl_attributes + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/ddl_attributes + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/ddl_attributes/storage_test/ticdc-ddl-attributes-test-10687?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3667c3900004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-gxw-8pkgb, pid:1249, start at 2024-05-06 19:11:25.415507225 +0800 CST m=+5.419249439 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:25.422 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:25.412 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:25.412 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 83-th check, retry later table changefeed_reconstruct.usertable exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table partition_table.finish_mark not exists for 14-th check, retry later table row_format.finish_mark not exists for 24-th check, retry later [Pipeline] // cache split_and_random_merge scale: 40 wait process 5546 exit for 1-th time... table multi_tables_ddl_test.finish_mark exists check table exists success + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process 5546 exit for 2-th time... + info='{ "upstream_id": 7365842438693335064, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449575357377150980, "checkpoint_time": "2024-05-06 19:11:26.479", "error": null }' + echo '{ "upstream_id": 7365842438693335064, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449575357377150980, "checkpoint_time": "2024-05-06 19:11:26.479", "error": null }' { "upstream_id": 7365842438693335064, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449575357377150980, "checkpoint_time": "2024-05-06 19:11:26.479", "error": null } ++ echo '{' '"upstream_id":' 7365842438693335064, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449575357377150980, '"checkpoint_time":' '"2024-05-06' '19:11:26.479",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365842438693335064, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449575357377150980, '"checkpoint_time":' '"2024-05-06' '19:11:26.479",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7365842438693335064, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449575357377150982, "checkpoint_time": "2024-05-06 19:11:26.479", "error": null }' + echo '{ "upstream_id": 7365842438693335064, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449575357377150982, "checkpoint_time": "2024-05-06 19:11:26.479", "error": null }' { "upstream_id": 7365842438693335064, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449575357377150982, "checkpoint_time": "2024-05-06 19:11:26.479", "error": null } ++ echo '{' '"upstream_id":' 7365842438693335064, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449575357377150982, '"checkpoint_time":' '"2024-05-06' '19:11:26.479",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365842438693335064, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449575357377150982, '"checkpoint_time":' '"2024-05-06' '19:11:26.479",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s + info='{ "upstream_id": 7365842438693335064, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449575349604581380, "checkpoint_time": "2024-05-06 19:10:56.829", "error": { "time": "2024-05-06T19:10:59.514125525+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7365842438693335064, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449575349604581380, "checkpoint_time": "2024-05-06 19:10:56.829", "error": { "time": "2024-05-06T19:10:59.514125525+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7365842438693335064, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449575349604581380, "checkpoint_time": "2024-05-06 19:10:56.829", "error": { "time": "2024-05-06T19:10:59.514125525+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '130', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ echo '{' '"upstream_id":' 7365842438693335064, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449575349604581380, '"checkpoint_time":' '"2024-05-06' '19:10:56.829",' '"error":' '{' '"time":' '"2024-05-06T19:10:59.514125525+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''130'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365842438693335064, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449575349604581380, '"checkpoint_time":' '"2024-05-06' '19:10:56.829",' '"error":' '{' '"time":' '"2024-05-06T19:10:59.514125525+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''130'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later + message='[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '130', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully wait process 5546 exit for 3-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3667c3900004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-gxw-8pkgb, pid:1249, start at 2024-05-06 19:11:25.415507225 +0800 CST m=+5.419249439 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:25.422 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:25.412 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:25.412 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3667c5240015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-gxw-8pkgb, pid:1337, start at 2024-05-06 19:11:25.552293293 +0800 CST m=+5.478705520 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:25.560 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:25.563 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:25.563 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 84-th check, retry later [Pipeline] } wait process cdc.test exit for 1-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5546) - No such process wait process 5546 exit for 4-th time... process 5546 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 19:11:28 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.58485850.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.finish_mark not exists for 15-th check, retry later table row_format.finish_mark exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // dir Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:11:29 CST 2024] <<<<<< run test case cdc success! >>>>>> wait process cdc.test exit for 2-th time... wait process cdc.test exit for 1-th time... [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... [Pipeline] // withCredentials [Pipeline] } wait process cdc.test exit for 3-th time... [Pipeline] // timeout [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3667ec100012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-722-gnh4g, pid:1198, start at 2024-05-06 19:11:28.03349022 +0800 CST m=+6.415244878 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:28.041 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:28.004 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:28.004 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3667ec100012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-722-gnh4g, pid:1198, start at 2024-05-06 19:11:28.03349022 +0800 CST m=+6.415244878 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:28.041 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:28.004 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:28.004 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3667edc00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-722-gnh4g, pid:1256, start at 2024-05-06 19:11:28.140045249 +0800 CST m=+6.447761282 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:28.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:28.112 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:28.112 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 6-th time, retry later [Pipeline] // stage cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:11:30 CST 2024] <<<<<< run test case csv_storage_multi_tables_ddl success! >>>>>> [Pipeline] } wait process cdc.test exit for 3-th time... [Pipeline] // container [Mon May 6 19:11:30 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=return(true);github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.26492651.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] } table ddl_manager.finish_mark not exists for 85-th check, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } table partition_table.finish_mark not exists for 16-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:11:30 CST 2024] <<<<<< run test case row_format success! >>>>>> [Pipeline] // podTemplate VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3668086c0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-4h8-1vqcr, pid:1369, start at 2024-05-06 19:11:29.825636862 +0800 CST m=+5.363015579 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:29.835 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:29.819 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:29.819 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // withEnv [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // stage [Pipeline] } check diff failed 7-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3667f6d80009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35j, pid:1284, start at 2024-05-06 19:11:28.701640075 +0800 CST m=+5.359841271 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:28.710 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:28.694 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:28.694 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3667f6d80009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35j, pid:1284, start at 2024-05-06 19:11:28.701640075 +0800 CST m=+5.359841271 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:28.710 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:28.694 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:28.694 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3667f8700006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35j, pid:1363, start at 2024-05-06 19:11:28.80254054 +0800 CST m=+5.364645394 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:28.809 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:28.796 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:28.796 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_resign/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_resign/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/61927129-6130-464d-8c83-ba58007b8cb5 {UpstreamID:7365842536789399572 Namespace:default ID:61927129-6130-464d-8c83-ba58007b8cb5 SinkURI:file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-25417?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:11:15.206764508 +0800 CST StartTs:449575354398670852 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003566f30 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575354411778055} {CheckpointTs:449575358475010051 MinTableBarrierTs:449575358737154052 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575358737154051, checkpointTs: 449575358475010052, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dacec534-dce4-4f1e-b2b7-0227c40b98e5 {"id":"dacec534-dce4-4f1e-b2b7-0227c40b98e5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993889} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9983b6a9 dacec534-dce4-4f1e-b2b7-0227c40b98e5 /tidb/cdc/default/default/changefeed/info/61927129-6130-464d-8c83-ba58007b8cb5 {"upstream-id":7365842536789399572,"namespace":"default","changefeed-id":"61927129-6130-464d-8c83-ba58007b8cb5","sink-uri":"file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-25417?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:11:15.206764508+08:00","start-ts":449575354398670852,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575354411778055} /tidb/cdc/default/default/changefeed/status/61927129-6130-464d-8c83-ba58007b8cb5 {"checkpoint-ts":449575358475010051,"min-table-barrier-ts":449575358737154052,"admin-job-type":0} /tidb/cdc/default/default/task/position/dacec534-dce4-4f1e-b2b7-0227c40b98e5/61927129-6130-464d-8c83-ba58007b8cb5 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365842536789399572 {"id":7365842536789399572,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/61927129-6130-464d-8c83-ba58007b8cb5 {UpstreamID:7365842536789399572 Namespace:default ID:61927129-6130-464d-8c83-ba58007b8cb5 SinkURI:file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-25417?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:11:15.206764508 +0800 CST StartTs:449575354398670852 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003566f30 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575354411778055} {CheckpointTs:449575358475010051 MinTableBarrierTs:449575358737154052 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575358737154051, checkpointTs: 449575358475010052, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dacec534-dce4-4f1e-b2b7-0227c40b98e5 {"id":"dacec534-dce4-4f1e-b2b7-0227c40b98e5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993889} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9983b6a9 dacec534-dce4-4f1e-b2b7-0227c40b98e5 /tidb/cdc/default/default/changefeed/info/61927129-6130-464d-8c83-ba58007b8cb5 {"upstream-id":7365842536789399572,"namespace":"default","changefeed-id":"61927129-6130-464d-8c83-ba58007b8cb5","sink-uri":"file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-25417?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:11:15.206764508+08:00","start-ts":449575354398670852,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575354411778055} /tidb/cdc/default/default/changefeed/status/61927129-6130-464d-8c83-ba58007b8cb5 {"checkpoint-ts":449575358475010051,"min-table-barrier-ts":449575358737154052,"admin-job-type":0} /tidb/cdc/default/default/task/position/dacec534-dce4-4f1e-b2b7-0227c40b98e5/61927129-6130-464d-8c83-ba58007b8cb5 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":n+ grep -q 'failed to get info:' ull,"warning":null} /tidb/cdc/default/default/upstream/7365842536789399572 {"id":7365842536789399572,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/61927129-6130-464d-8c83-ba58007b8cb5 {UpstreamID:7365842536789399572 Namespace:default ID:61927129-6130-464d-8c83-ba58007b8cb5 SinkURI:file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-25417?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:11:15.206764508 +0800 CST StartTs:449575354398670852 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003566f30 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575354411778055} {CheckpointTs:449575358475010051 MinTableBarrierTs:449575358737154052 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575358737154051, checkpointTs: 449575358475010052, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dacec534-dce4-4f1e-b2b7-0227c40b98e5 {"id":"dacec534-dce4-4f1e-b2b7-0227c40b98e5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993889} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9983b6a9 dacec534-dce4-4f1e-b2b7-0227c40b98e5 /tidb/cdc/default/default/changefeed/info/61927129-6130-464d-8c83-ba58007b8cb5 {"upstream-id":7365842536789399572,"namespace":"default","changefeed-id":"61927129-6130-464d-8c83-ba58007b8cb5","sink-uri":"file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-25417?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:11:15.206764508+08:00","start-ts":449575354398670852,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575354411778055} /tidb/cdc/default/default/changefeed/status/61927129-6130-464d-8c83-ba58007b8cb5 {"checkpoint-ts":449575358475010051,"min-table-barrier-ts":449575358737154052,"admin-job-type":0} /tidb/cdc/default/default/task/position/dacec534-dce4-4f1e-b2b7-0227c40b98e5/61927129-6130-464d-8c83-ba58007b8cb5 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365842536789399572 {"id":7365842536789399572,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:11:31 CST 2024] <<<<<< run test case sink_retry success! >>>>>> table ddl_manager.finish_mark not exists for 86-th check, retry later table ddl_attributes.attributes_t1_new not exists for 1-th check, retry later [Mon May 6 19:11:32 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.25332535.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3668086c0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-4h8-1vqcr, pid:1369, start at 2024-05-06 19:11:29.825636862 +0800 CST m=+5.363015579 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:29.835 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:29.819 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:29.819 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36680ad0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-4h8-1vqcr, pid:1453, start at 2024-05-06 19:11:29.984656902 +0800 CST m=+5.452441327 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:29.994 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:29.972 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:29.972 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 17-th check, retry later "id": "dacec534-dce4-4f1e-b2b7-0227c40b98e5", "cluster-id": "default" run task successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:33 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/052df61b-a5cc-4649-9853-82abe53a09e3 {"id":"052df61b-a5cc-4649-9853-82abe53a09e3","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993890} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99cdb5cd 052df61b-a5cc-4649-9853-82abe53a09e3 /tidb/cdc/default/default/upstream/7365842611784079181 {"id":7365842611784079181,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/052df61b-a5cc-4649-9853-82abe53a09e3 {"id":"052df61b-a5cc-4649-9853-82abe53a09e3","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993890} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99cdb5cd 052df61b-a5cc-4649-9853-82abe53a09e3 /tidb/cdc/default/default/upstream/7365842611784079181 {"id":7365842611784079181,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/052df61b-a5cc-4649-9853-82abe53a09e3 {"id":"052df61b-a5cc-4649-9853-82abe53a09e3","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993890} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99cdb5cd 052df61b-a5cc-4649-9853-82abe53a09e3 /tidb/cdc/default/default/upstream/7365842611784079181 {"id":7365842611784079181,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.cli.2704.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13577?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 table ddl_attributes.attributes_t1_new not exists for 2-th check, retry later *************************** 1. row *************************** count(distinct region_id): 1 Create changefeed successfully! ID: 1a77b9f2-eda7-44db-8247-bbfa567868d4 Info: {"upstream_id":7365842611784079181,"namespace":"default","id":"1a77b9f2-eda7-44db-8247-bbfa567868d4","sink_uri":"file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13577?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:11:33.856672012+08:00","start_ts":449575359286345732,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575359286345732,"checkpoint_ts":449575359286345732,"checkpoint_time":"2024-05-06 19:11:33.762"} PASS [Mon May 6 19:11:33 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.26282630.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 coverage: 2.5% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 87-th check, retry later check diff failed 8-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36684e680002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m, pid:3865, start at 2024-05-06 19:11:34.299517231 +0800 CST m=+5.471971145 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:34.308 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:34.298 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:34.298 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 18-th check, retry later capture_id: dacec534-dce4-4f1e-b2b7-0227c40b98e5 check_processor_table_count http://127.0.0.1:2379 61927129-6130-464d-8c83-ba58007b8cb5 dacec534-dce4-4f1e-b2b7-0227c40b98e5 1 run task successfully < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:35 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e4007ad-c9b3-46c4-9c38-8ac9bc665e8f {"id":"8e4007ad-c9b3-46c4-9c38-8ac9bc665e8f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993892} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99cbedcc 8e4007ad-c9b3-46c4-9c38-8ac9bc665e8f /tidb/cdc/default/default/upstream/7365842612003849854 {"id":7365842612003849854,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e4007ad-c9b3-46c4-9c38-8ac9bc665e8f {"id":"8e4007ad-c9b3-46c4-9c38-8ac9bc665e8f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993892} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99cbedcc 8e4007ad-c9b3-46c4-9c38-8ac9bc665e8f /tidb/cdc/default/default/upstream/7365842612003849854 {"id":7365842612003849854,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e4007ad-c9b3-46c4-9c38-8ac9bc665e8f {"id":"8e4007ad-c9b3-46c4-9c38-8ac9bc665e8f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993892} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99cbedcc 8e4007ad-c9b3-46c4-9c38-8ac9bc665e8f /tidb/cdc/default/default/upstream/7365842612003849854 {"id":7365842612003849854,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + set +x table ddl_attributes.attributes_t1_new not exists for 3-th check, retry later + workdir=/tmp/tidb_cdc_test/changefeed_finish + workdir=/tmp/tidb_cdc_test/processor_resolved_ts_fallback + sink_uri='file:///tmp/tidb_cdc_test/changefeed_finish/storage_test/ticdc-changefeed-pause-resume-13048?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + sink_uri='file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13577?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/changefeed_finish ++ date + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/processor_resolved_ts_fallback ++ date + echo '[Mon May 6 19:11:35 CST 2024] <<<<<< START storage consumer in changefeed_finish case >>>>>>' + echo '[Mon May 6 19:11:35 CST 2024] <<<<<< START storage consumer in processor_resolved_ts_fallback case >>>>>>' [Mon May 6 19:11:35 CST 2024] <<<<<< START storage consumer in changefeed_finish case >>>>>> [Mon May 6 19:11:35 CST 2024] <<<<<< START storage consumer in processor_resolved_ts_fallback case >>>>>> + cd /tmp/tidb_cdc_test/changefeed_finish + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/processor_resolved_ts_fallback + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/changefeed_finish + set +x + cd /tmp/tidb_cdc_test/processor_resolved_ts_fallback + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13577?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/changefeed_finish/storage_test/ticdc-changefeed-pause-resume-13048?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' check_processor_table_count http://127.0.0.1:2379 61927129-6130-464d-8c83-ba58007b8cb5 dacec534-dce4-4f1e-b2b7-0227c40b98e5 0 cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table processor_resolved_ts_fallback.t1 not exists for 1-th check, retry later [Mon May 6 19:11:35 CST 2024] <<<<<< START cdc server in batch_add_table case >>>>>> check diff successfully table count 1 does equal to expected count 0 run task failed 1-th time, retry later + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.27152717.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 88-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36684e680002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m, pid:3865, start at 2024-05-06 19:11:34.299517231 +0800 CST m=+5.471971145 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:34.308 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:34.298 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:34.298 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36684e6c0001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m, pid:3942, start at 2024-05-06 19:11:34.299496983 +0800 CST m=+5.413617174 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:34.307 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:34.299 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:34.299 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table partition_table.finish_mark not exists for 19-th check, retry later check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:36 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ede3390c-53ad-4d78-93ef-4778be66d623 {"id":"ede3390c-53ad-4d78-93ef-4778be66d623","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993894} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99d8c8ce ede3390c-53ad-4d78-93ef-4778be66d623 /tidb/cdc/default/default/upstream/7365842627759809673 {"id":7365842627759809673,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ede3390c-53ad-4d78-93ef-4778be66d623 {"id":"ede3390c-53ad-4d78-93ef-4778be66d623","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993894} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99d8c8ce ede3390c-53ad-4d78-93ef-4778be66d623 /tidb/cdc/default/default/upstream/7365842627759809673 {"id":7365842627759809673,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ede3390c-53ad-4d78-93ef-4778be66d623 {"id":"ede3390c-53ad-4d78-93ef-4778be66d623","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993894} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99d8c8ce ede3390c-53ad-4d78-93ef-4778be66d623 /tidb/cdc/default/default/upstream/7365842627759809673 {"id":7365842627759809673,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.cli.2724.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-30730?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Create changefeed successfully! ID: dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 Info: {"upstream_id":7365842627759809673,"namespace":"default","id":"dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8","sink_uri":"file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-30730?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:11:37.363513609+08:00","start_ts":449575360199131139,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575360199131139,"checkpoint_ts":449575360199131139,"checkpoint_time":"2024-05-06 19:11:37.244"} PASS table ddl_attributes.attributes_t1_new not exists for 4-th check, retry later coverage: 2.5% of statements in github.com/pingcap/tiflow/... table processor_resolved_ts_fallback.t1 not exists for 2-th check, retry later check_processor_table_count http://127.0.0.1:2379 61927129-6130-464d-8c83-ba58007b8cb5 dacec534-dce4-4f1e-b2b7-0227c40b98e5 0 run task successfully table ddl_manager.finish_mark not exists for 89-th check, retry later table partition_table.finish_mark not exists for 20-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:38 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e3308ae-7dc1-4d35-a183-e444bab0d1ea {"id":"4e3308ae-7dc1-4d35-a183-e444bab0d1ea","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993896} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99d94519 4e3308ae-7dc1-4d35-a183-e444bab0d1ea /tidb/cdc/default/default/upstream/7365842635531490575 {"id":7365842635531490575,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e3308ae-7dc1-4d35-a183-e444bab0d1ea {"id":"4e3308ae-7dc1-4d35-a183-e444bab0d1ea","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993896} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99d94519 4e3308ae-7dc1-4d35-a183-e444bab0d1ea /tidb/cdc/default/default/upstream/7365842635531490575 {"id":7365842635531490575,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e3308ae-7dc1-4d35-a183-e444bab0d1ea {"id":"4e3308ae-7dc1-4d35-a183-e444bab0d1ea","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993896} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99d94519 4e3308ae-7dc1-4d35-a183-e444bab0d1ea /tidb/cdc/default/default/upstream/7365842635531490575 {"id":7365842635531490575,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.2776.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/batch_add_table/storage_test/ticdc-batch-add-table-test-11353?protocol=canal-json&enable-tidb-extension=true' + set +x + workdir=/tmp/tidb_cdc_test/owner_resign + sink_uri='file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-30730?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/owner_resign ++ date + echo '[Mon May 6 19:11:38 CST 2024] <<<<<< START storage consumer in owner_resign case >>>>>>' [Mon May 6 19:11:38 CST 2024] <<<<<< START storage consumer in owner_resign case >>>>>> + cd /tmp/tidb_cdc_test/owner_resign + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/owner_resign + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/owner_resign/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-30730?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' split_and_random_merge scale: 80 valid ~~~ running cdc Failed to start cdc, the usage tips should be printed 1st test case cdc_server_tips success! try an INVALID cdc server command [Mon May 6 19:11:37 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ true != \n\o ]] + set +x + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.55315533.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default --pd None ***************** properties ***************** "readproportion"="0" "threadcount"="4" "mysql.host"="127.0.0.1" "scanproportion"="0" "insertproportion"="0" "mysql.port"="4000" "readallfields"="true" "operationcount"="0" "recordcount"="50" "requestdistribution"="uniform" "mysql.user"="root" "workload"="core" "mysql.db"="changefeed_reconstruct" "dotransactions"="false" "updateproportion"="0" ********************************************** Run finished, takes 19.544339ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3269.9, Avg(us): 1544, Min(us): 996, Max(us): 5255, 95th(us): 5000, 99th(us): 6000 table changefeed_reconstruct.usertable exists table ddl_attributes.attributes_t1_new not exists for 5-th check, retry later Create changefeed successfully! ID: 9855f201-4996-4127-b9a3-6133707727e8 Info: {"upstream_id":7365842635531490575,"namespace":"default","id":"9855f201-4996-4127-b9a3-6133707727e8","sink_uri":"file:///tmp/tidb_cdc_test/batch_add_table/storage_test/ticdc-batch-add-table-test-11353?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:11:39.380687263+08:00","start_ts":449575360729972741,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575360729972741,"checkpoint_ts":449575360729972741,"checkpoint_time":"2024-05-06 19:11:39.269"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check diff failed 2-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5242.out cli tso query --pd=http://127.0.0.1:2379 table processor_resolved_ts_fallback.t1 not exists for 3-th check, retry later check diff failed 1-th time, retry later cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table owner_resign.t1 not exists for 1-th check, retry later check diff failed 1-th time, retry later table ddl_manager.finish_mark exists check diff successfully table partition_table.finish_mark exists + set +x + workdir=/tmp/tidb_cdc_test/batch_add_table + sink_uri='file:///tmp/tidb_cdc_test/batch_add_table/storage_test/ticdc-batch-add-table-test-11353?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/batch_add_table ++ date + echo '[Mon May 6 19:11:40 CST 2024] <<<<<< START storage consumer in batch_add_table case >>>>>>' [Mon May 6 19:11:40 CST 2024] <<<<<< START storage consumer in batch_add_table case >>>>>> + cd /tmp/tidb_cdc_test/batch_add_table + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/batch_add_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/batch_add_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/batch_add_table/storage_test/ticdc-batch-add-table-test-11353?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' check diff successfully wait process cdc.test exit for 1-th time... + set +x + tso='449575360803110913 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575360803110913 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table batch_add_table.finish_mark not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... [Mon May 6 19:11:41 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.52915293.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... wait process cdc.test exit for 1-th time... table processor_resolved_ts_fallback.t1 not exists for 4-th check, retry later wait process cdc.test exit for 3-th time... table ddl_attributes.attributes_t1_new not exists for 6-th check, retry later table owner_resign.t1 not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... check diff failed 2-th time, retry later wait process cdc.test exit for 3-th time... check diff failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:11:42 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_partition_table/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:11:42 CST 2024] <<<<<< run test case canal_json_storage_partition_table success! >>>>>> table batch_add_table.finish_mark not exists for 2-th check, retry later check diff failed 4-th time, retry later table processor_resolved_ts_fallback.t1 not exists for 5-th check, retry later table ddl_attributes.attributes_t1_new not exists for 7-th check, retry later *************************** 1. row *************************** count(distinct region_id): 3 check diff failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:44 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/84a27f5c-8d5b-415c-92b1-9c9a1d6a737f {"id":"84a27f5c-8d5b-415c-92b1-9c9a1d6a737f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993901} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99e645f6 84a27f5c-8d5b-415c-92b1-9c9a1d6a737f /tidb/cdc/default/default/upstream/7365842643524179989 {"id":7365842643524179989,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/84a27f5c-8d5b-415c-92b1-9c9a1d6a737f {"id":"84a27f5c-8d5b-415c-92b1-9c9a1d6a737f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993901} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99e645f6 84a27f5c-8d5b-415c-92b1-9c9a1d6a737f /tidb/cdc/default/default/upstream/7365842643524179989 {"id":7365842643524179989,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/84a27f5c-8d5b-415c-92b1-9c9a1d6a737f {"id":"84a27f5c-8d5b-415c-92b1-9c9a1d6a737f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993901} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99e645f6 84a27f5c-8d5b-415c-92b1-9c9a1d6a737f /tidb/cdc/default/default/upstream/7365842643524179989 {"id":7365842643524179989,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5347.out cli changefeed create --start-ts=449575360803110913 '--sink-uri=file:///tmp/tidb_cdc_test/ddl_puller_lag/storage_test/ticdc-ddl-puller-lag-test-24714?protocol=canal-json&enable-tidb-extension=true' table owner_resign.t1 not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 3-th time, retry later Create changefeed successfully! ID: 9eac988a-bad3-4a09-8463-79393fbdebae Info: {"upstream_id":7365842643524179989,"namespace":"default","id":"9eac988a-bad3-4a09-8463-79393fbdebae","sink_uri":"file:///tmp/tidb_cdc_test/ddl_puller_lag/storage_test/ticdc-ddl-puller-lag-test-24714?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:11:44.828131331+08:00","start_ts":449575360803110913,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575360803110913,"checkpoint_ts":449575360803110913,"checkpoint_time":"2024-05-06 19:11:39.548"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table batch_add_table.finish_mark not exists for 3-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table ddl_attributes.attributes_t1_new not exists for 8-th check, retry later check diff failed 5-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/csv_storage_partition_table Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table processor_resolved_ts_fallback.t1 exists [Mon May 6 19:11:45 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.28602862.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table owner_resign.t1 not exists for 4-th check, retry later + set +x + workdir=/tmp/tidb_cdc_test/ddl_puller_lag + sink_uri='file:///tmp/tidb_cdc_test/ddl_puller_lag/storage_test/ticdc-ddl-puller-lag-test-24714?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/ddl_puller_lag ++ date + echo '[Mon May 6 19:11:46 CST 2024] <<<<<< START storage consumer in ddl_puller_lag case >>>>>>' [Mon May 6 19:11:46 CST 2024] <<<<<< START storage consumer in ddl_puller_lag case >>>>>> + cd /tmp/tidb_cdc_test/ddl_puller_lag + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/ddl_puller_lag + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/ddl_puller_lag/storage_test/ticdc-ddl-puller-lag-test-24714?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... check diff failed 4-th time, retry later check diff successfully Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 1-th time... table batch_add_table.finish_mark not exists for 4-th check, retry later table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... check diff successfully cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:11:47 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table owner_resign.t1 not exists for 5-th check, retry later check diff failed 5-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:48 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/1a77b9f2-eda7-44db-8247-bbfa567868d4 {UpstreamID:7365842611784079181 Namespace:default ID:1a77b9f2-eda7-44db-8247-bbfa567868d4 SinkURI:file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13577?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:11:33.856672012 +0800 CST StartTs:449575359286345732 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00256a240 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575359299190792} {CheckpointTs:449575363034742798 MinTableBarrierTs:449575363034742798 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/052df61b-a5cc-4649-9853-82abe53a09e3 {"id":"052df61b-a5cc-4649-9853-82abe53a09e3","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993890} /tidb/cdc/default/__cdc_meta__/capture/5d27475e-8366-45e2-9d7f-59e72274d3ba {"id":"5d27475e-8366-45e2-9d7f-59e72274d3ba","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993906} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99cdb5cd 052df61b-a5cc-4649-9853-82abe53a09e3 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99cdb688 5d27475e-8366-45e2-9d7f-59e72274d3ba /tidb/cdc/default/default/changefeed/info/1a77b9f2-eda7-44db-8247-bbfa567868d4 {"upstream-id":7365842611784079181,"namespace":"default","changefeed-id":"1a77b9f2-eda7-44db-8247-bbfa567868d4","sink-uri":"file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13577?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:11:33.856672012+08:00","start-ts":449575359286345732,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575359299190792} /tidb/cdc/default/default/changefeed/status/1a77b9f2-eda7-44db-8247-bbfa567868d4 {"checkpoint-ts":449575363034742798,"min-table-barrier-ts":449575363034742798,"admin-job-type":0} /tidb/cdc/default/default/task/position/052df61b-a5cc-4649-9853-82abe53a09e3/1a77b9f2-eda7-44db-8247-bbfa567868d4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5d27475e-8366-45e2-9d7f-59e72274d3ba/1a77b9f2-eda7-44db-8247-bbfa567868d4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365842611784079181 {"id":7365842611784079181,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/1a77b9f2-eda7-44db-8247-bbfa567868d4 {UpstreamID:7365842611784079181 Namespace:default ID:1a77b9f2-eda7-44db-8247-bbfa567868d4 SinkURI:file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13577?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:11:33.856672012 +0800 CST StartTs:449575359286345732 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00256a240 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575359299190792} {CheckpointTs:449575363034742798 MinTableBarrierTs:449575363034742798 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/052df61b-a5cc-4649-9853-82abe53a09e3 {"id":"052df61b-a5cc-4649-9853-82abe53a09e3","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993890} /tidb/cdc/default/__cdc_meta__/capture/5d27475e-8366-45e2-9d7f-59e72274d3ba {"id":"5d27475e-8366-45e2-9d7f-59e72274d3ba","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993906} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99cdb5cd 052df61b-a5cc-4649-9853-82abe53a09e3 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99cdb688 5d27475e-8366-45e2-9d7f-59e72274d3ba /tidb/cdc/default/default/changefeed/info/1a77b9f2-eda7-44db-8247-bbfa567868d4 {"upstream-id":7365842611784079181,"namespace":"default","changefeed-id":"1a77b9f2-eda7-44db-8247-bbfa567868d4","sink-uri":"file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13577?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:11:33.856672012+08:00","start-ts":449575359286345732,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575359299190792} /tidb/cdc/default/default/changefeed/status/1a77b+ grep -q 'failed to get info:' 9f2-eda7-44db-8247-bbfa567868d4 {"checkpoint-ts":449575363034742798,"min-table-barrier-ts":449575363034742798,"admin-job-type":0} /tidb/cdc/default/default/task/position/052df61b-a5cc-4649-9853-82abe53a09e3/1a77b9f2-eda7-44db-8247-bbfa567868d4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5d27475e-8366-45e2-9d7f-59e72274d3ba/1a77b9f2-eda7-44db-8247-bbfa567868d4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365842611784079181 {"id":7365842611784079181,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/1a77b9f2-eda7-44db-8247-bbfa567868d4 {UpstreamID:7365842611784079181 Namespace:default ID:1a77b9f2-eda7-44db-8247-bbfa567868d4 SinkURI:file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13577?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:11:33.856672012 +0800 CST StartTs:449575359286345732 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00256a240 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575359299190792} {CheckpointTs:449575363034742798 MinTableBarrierTs:449575363034742798 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/052df61b-a5cc-4649-9853-82abe53a09e3 {"id":"052df61b-a5cc-4649-9853-82abe53a09e3","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993890} /tidb/cdc/default/__cdc_meta__/capture/5d27475e-8366-45e2-9d7f-59e72274d3ba {"id":"5d27475e-8366-45e2-9d7f-59e72274d3ba","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993906} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99cdb5cd 052df61b-a5cc-4649-9853-82abe53a09e3 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99cdb688 5d27475e-8366-45e2-9d7f-59e72274d3ba /tidb/cdc/default/default/changefeed/info/1a77b9f2-eda7-44db-8247-bbfa567868d4 {"upstream-id":7365842611784079181,"namespace":"default","changefeed-id":"1a77b9f2-eda7-44db-8247-bbfa567868d4","sink-uri":"file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13577?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:11:33.856672012+08:00","start-ts":449575359286345732,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575359299190792} /tidb/cdc/default/default/changefeed/status/1a77b9f2-eda7-44db-8247-bbfa567868d4 {"checkpoint-ts":449575363034742798,"min-table-barrier-ts":449575363034742798,"admin-job-type":0} /tidb/cdc/default/default/task/position/052df61b-a5cc-4649-9853-82abe53a09e3/1a77b9f2-eda7-44db-8247-bbfa567868d4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5d27475e-8366-45e2-9d7f-59e72274d3ba/1a77b9f2-eda7-44db-8247-bbfa567868d4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365842611784079181 {"id":7365842611784079181,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_resolved_ts_fallback.t2 not exists for 1-th check, retry later table batch_add_table.finish_mark not exists for 5-th check, retry later table region_merge.t1 exists check diff failed 1-th time, retry later table ddl_attributes.finish_mark not exists for 2-th check, retry later table owner_resign.t1 exists check diff failed 1-th time, retry later check diff failed 6-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table batch_add_table.finish_mark not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later table processor_resolved_ts_fallback.t2 not exists for 2-th check, retry later table ddl_attributes.finish_mark not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 7-th time, retry later table batch_add_table.finish_mark exists VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3669622c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-2p4-slmd8, pid:4478, start at 2024-05-06 19:11:51.997720801 +0800 CST m=+5.311482732 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:52.005 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:51.997 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:51.997 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later check diff successfully table processor_resolved_ts_fallback.t2 not exists for 3-th check, retry later table ddl_attributes.finish_mark not exists for 4-th check, retry later wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 2-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:11:53 CST 2024] <<<<<< run test case multi_source success! >>>>>> check diff failed 3-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:11:54 CST 2024] <<<<<< run test case batch_add_table success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3669622c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-2p4-slmd8, pid:4478, start at 2024-05-06 19:11:51.997720801 +0800 CST m=+5.311482732 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:52.005 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:51.997 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:51.997 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366962f80011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-2p4-slmd8, pid:4561, start at 2024-05-06 19:11:52.018541064 +0800 CST m=+5.274003281 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:52.026 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:51.998 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:51.998 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/split_region/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... table processor_resolved_ts_fallback.t2 not exists for 4-th check, retry later Starting Upstream TiDB... table ddl_attributes.finish_mark not exists for 5-th check, retry later Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... check diff failed 4-th time, retry later wait process cdc.test exit for 3-th time... check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:11:56 CST 2024] <<<<<< run test case changefeed_pause_resume success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc already exists) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_resolved_ts_fallback.t2 exists table processor_resolved_ts_fallback.t3 exists cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^2$' 2 run task successfully invalid ~~~ running cdc Failed to start cdc, the usage tips should be printed 2nd test case cdc_server_tips success! [Mon May 6 19:11:57 CST 2024] <<<<<< run all test cases cdc_server_tips success! >>>>>> [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } killing cdc1: 2654 cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 1-th time, retry later table ddl_attributes.finish_mark not exists for 6-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.5818.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] // withCredentials ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/multi_tables_ddl/run.sh using Sink-Type: storage... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 [Pipeline] // timeout [Pipeline] } wait process cdc.test exit for 1-th time... [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend [Pipeline] // node [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // podTemplate [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide [Pipeline] } + set +x + tso='449575365469274113 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575365469274113 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x check diff failed 5-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:11:58 CST 2024] <<<<<< run test case region_merge success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 19:11:59 CST 2024] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.58675869.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCE1E22E0B0400 < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 11:11:59 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Mon May 6 19:11:59 CST 2024] <<<<<< run test case multi_tables_ddl success! >>>>>> Exiting on signal: INTERRUPT table ddl_attributes.finish_mark not exists for 7-th check, retry later cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 2-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully [Mon May 6 19:12:00 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerDontUpdateCheckpoint=return(true)' + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.31093111.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3669ccfc000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9, pid:10393, start at 2024-05-06 19:11:58.797996262 +0800 CST m=+8.081031617 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:58.807 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:58.783 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:58.783 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3669ccfc000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9, pid:10393, start at 2024-05-06 19:11:58.797996262 +0800 CST m=+8.081031617 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:58.807 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:58.783 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:58.783 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3669cdc0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-5ln-kfhk9, pid:10470, start at 2024-05-06 19:11:58.849616431 +0800 CST m=+8.078496405 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:58.863 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:58.832 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:58.832 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/csv_storage_partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/csv_storage_partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/csv_storage_partition_table/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/csv_storage_partition_table/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/csv_storage_partition_table/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_attributes.finish_mark not exists for 8-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3669edac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5, pid:3969, start at 2024-05-06 19:12:00.91545592 +0800 CST m=+5.350296014 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:00.922 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:00.925 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:00.925 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3669fc780014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms, pid:3903, start at 2024-05-06 19:12:01.868974181 +0800 CST m=+7.738516218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:01.882 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:01.872 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:01.872 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:02 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8753e846-6b9c-40f3-880d-453220574f57 {"id":"8753e846-6b9c-40f3-880d-453220574f57","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993919} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a3019f8 8753e846-6b9c-40f3-880d-453220574f57 /tidb/cdc/default/default/upstream/7365842728618086932 {"id":7365842728618086932,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8753e846-6b9c-40f3-880d-453220574f57 {"id":"8753e846-6b9c-40f3-880d-453220574f57","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993919} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a3019f8 8753e846-6b9c-40f3-880d-453220574f57 /tidb/cdc/default/default/upstream/7365842728618086932 {"id":7365842728618086932,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8753e846-6b9c-40f3-880d-453220574f57 {"id":"8753e846-6b9c-40f3-880d-453220574f57","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993919} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a3019f8 8753e846-6b9c-40f3-880d-453220574f57 /tidb/cdc/default/default/upstream/7365842728618086932 {"id":7365842728618086932,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.5920.out cli changefeed create --start-ts=449575365469274113 '--sink-uri=file:///tmp/tidb_cdc_test/split_region/storage_test/ticdc-split-region-test-4952?protocol=canal-json&enable-tidb-extension=true' --config /tmp/tidb_cdc_test/split_region/pulsar_test.toml Create changefeed successfully! ID: c7ec4b9a-1173-436a-9b49-239683089c3f Info: {"upstream_id":7365842728618086932,"namespace":"default","id":"c7ec4b9a-1173-436a-9b49-239683089c3f","sink_uri":"file:///tmp/tidb_cdc_test/split_region/storage_test/ticdc-split-region-test-4952?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:12:02.63513426+08:00","start_ts":449575365469274113,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575365469274113,"checkpoint_ts":449575365469274113,"checkpoint_time":"2024-05-06 19:11:57.348"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [Mon May 6 19:12:03 CST 2024] <<<<<< START cdc server in csv_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.1173911741.out server --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_attributes.finish_mark not exists for 9-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3669edac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5, pid:3969, start at 2024-05-06 19:12:00.91545592 +0800 CST m=+5.350296014 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:00.922 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:00.925 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:00.925 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3669fae4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5, pid:4051, start at 2024-05-06 19:12:01.733066333 +0800 CST m=+6.098734259 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:01.742 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:01.721 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:01.721 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + workdir=/tmp/tidb_cdc_test/split_region + sink_uri='file:///tmp/tidb_cdc_test/split_region/storage_test/ticdc-split-region-test-4952?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/split_region ++ date + echo '[Mon May 6 19:12:04 CST 2024] <<<<<< START storage consumer in split_region case >>>>>>' [Mon May 6 19:12:04 CST 2024] <<<<<< START storage consumer in split_region case >>>>>> + cd /tmp/tidb_cdc_test/split_region + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/split_region + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/split_region/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/split_region/storage_test/ticdc-split-region-test-4952?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table split_region.test1 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:03 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 {UpstreamID:7365842627759809673 Namespace:default ID:dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 SinkURI:file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-30730?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:11:37.363513609 +0800 CST StartTs:449575360199131139 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001388c60 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575360225345541} {CheckpointTs:449575367054196746 MinTableBarrierTs:449575367054196746 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/df316c84-0f64-4434-8117-edae48efd545 {"id":"df316c84-0f64-4434-8117-edae48efd545","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993921} /tidb/cdc/default/__cdc_meta__/capture/ede3390c-53ad-4d78-93ef-4778be66d623 {"id":"ede3390c-53ad-4d78-93ef-4778be66d623","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993894} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99d8c8ce ede3390c-53ad-4d78-93ef-4778be66d623 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99d8ca15 df316c84-0f64-4434-8117-edae48efd545 /tidb/cdc/default/default/changefeed/info/dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 {"upstream-id":7365842627759809673,"namespace":"default","changefeed-id":"dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8","sink-uri":"file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-30730?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:11:37.363513609+08:00","start-ts":449575360199131139,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575360225345541} /tidb/cdc/default/default/changefeed/status/dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 {"checkpoint-ts":449575367054196746,"min-table-barrier-ts":449575367054196746,"admin-job-type":0} /tidb/cdc/default/default/task/position/df316c84-0f64-4434-8117-edae48efd545/dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ede3390c-53ad-4d78-93ef-4778be66d623/dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365842627759809673 {"id":7365842627759809673,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 {UpstreamID:7365842627759809673 Namespace:default ID:dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 SinkURI:file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-30730?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:11:37.363513609 +0800 CST StartTs:449575360199131139 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001388c60 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575360225345541} {CheckpointTs:449575367054196746 MinTableBarrierTs:449575367054196746 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/df316c84-0f64-4434-8117-edae48efd545 {"id":"df316c84-0f64-4434-8117-edae48efd545","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993921} /tidb/cdc/default/__cdc_meta__/capture/ede3390c-53ad-4d78-93ef-4778be66d623 {"id":"ede3390c-53ad-4d78-93ef-4778be66d623","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993894} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99d8c8ce ede3390c-53ad-4d78-93ef-4778be66d623 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99d8ca15 df316c84-0f64-4434-8117-edae48efd545 /tidb/cdc/default/default/changefeed/info/dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 {"upstream-id":7365842627759809673,"namespace":"default","changefeed-id":"dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8","sink-uri":"file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-30730?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:11:37.363513609+08:00","start-ts":449575360199131139,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575360225345541} /tidb/cdc/default/default/changefeed/status/dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 {"checkpoint-ts":449575367054196746,"min-table-barrier-ts":449575367054196746,"admin-job-type":0} /tidb/cdc/default/default/task/position/df316c84-0f64-4434-8117-edae48efd545/dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ede3390c-53ad-4d78-93ef-4778be66d623/dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365842627759809673 {"id":7365842627759809673,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 {UpstreamID:7365842627759809673 Namespace:default ID:dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 SinkURI:file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-30730?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:11:37.363513609 +0800 CST StartTs:449575360199131139 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001388c60 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575360225345541} {CheckpointTs:449575367054196746 MinTableBarrierTs:449575367054196746 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/df316c84-0f64-4434-8117-edae48efd545 {"id":"df316c84-0f64-4434-8117-edae48efd545","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993921} /tidb/cdc/default/__cdc_meta__/capture/ede3390c-53ad-4d78-93ef-4778be66d623 {"id":"ede3390c-53ad-4d78-93ef-4778be66d623","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993894} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99d8c8ce ede3390c-53ad-4d78-93ef-4778be66d623 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99d8ca15 df316c84-0f64-4434-8117-edae48efd545 /tidb/cdc/default/default/changefeed/info/dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 {"upstream-id":7365842627759809673,"namespace":"default","changefeed-id":"dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8","sink-uri":"file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-30730?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:11:37.363513609+08:00","start-ts":449575360199131139,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575360225345541} /tidb/cdc/default/default/changefeed/status/dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 {"checkpoint-ts":449575367054196746,"min-table-barrier-ts":449575367054196746,"admin-job-type":0} /tidb/cdc/default/default/task/position/df316c84-0f64-4434-8117-edae48efd545/dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ede3390c-53ad-4d78-93ef-4778be66d623/dd37c75c-c0c0-4344-b8b4-27ab72c8e1a8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365842627759809673 {"id":7365842627759809673,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^2$' 2 run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3669fc780014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms, pid:3903, start at 2024-05-06 19:12:01.868974181 +0800 CST m=+7.738516218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:01.882 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:01.872 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:01.872 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3669fe3c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms, pid:3969, start at 2024-05-06 19:12:01.936606278 +0800 CST m=+7.722360954 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:01.944 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:01.935 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:01.935 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_attributes.finish_mark not exists for 10-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5363.out cli tso query --pd=http://127.0.0.1:2379 table split_region.test1 not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:06 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a15075d-43a4-47db-9ebb-bd00b6639491 {"id":"5a15075d-43a4-47db-9ebb-bd00b6639491","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993923} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a419bd2 5a15075d-43a4-47db-9ebb-bd00b6639491 /tidb/cdc/default/default/upstream/7365842741345146855 {"id":7365842741345146855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a15075d-43a4-47db-9ebb-bd00b6639491 {"id":"5a15075d-43a4-47db-9ebb-bd00b6639491","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993923} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a419bd2 5a15075d-43a4-47db-9ebb-bd00b6639491 /tidb/cdc/default/default/upstream/7365842741345146855 {"id":7365842741345146855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a15075d-43a4-47db-9ebb-bd00b6639491 {"id":"5a15075d-43a4-47db-9ebb-bd00b6639491","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993923} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a419bd2 5a15075d-43a4-47db-9ebb-bd00b6639491 /tidb/cdc/default/default/upstream/7365842741345146855 {"id":7365842741345146855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:12:06 CST 2024] <<<<<< START cdc server in csv_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.1179711799.out server --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/multi_rocks/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5208.out cli tso query --pd=http://127.0.0.1:2379 table ddl_attributes.finish_mark exists + set +x + tso='449575367796326401 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575367796326401 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:12:07 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.54015403.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } check diff failed 1-th time, retry later [Pipeline] // timeout wait process cdc.test exit for 1-th time... [Pipeline] } table split_region.test1 not exists for 3-th check, retry later [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // withEnv + set +x + tso='449575367979040769 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575367979040769 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/cli_with_auth/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... [Pipeline] // node [Pipeline] } wait process cdc.test exit for 3-th time... ***************** properties ***************** "readallfields"="true" "readproportion"="0" "threadcount"="4" "insertproportion"="0" "scanproportion"="0" "updateproportion"="0" "mysql.port"="4000" "dotransactions"="false" "operationcount"="0" "mysql.user"="root" "mysql.host"="127.0.0.1" "recordcount"="20" "mysql.db"="changefeed_error" "workload"="core" "requestdistribution"="uniform" ********************************************** Run finished, takes 14.484663ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2455.0, Avg(us): 2683, Min(us): 1327, Max(us): 6420, 95th(us): 7000, 99th(us): 7000 [Mon May 6 19:12:09 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.52635265.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // podTemplate + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:09 GMT < Content-Length: 1275 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a15075d-43a4-47db-9ebb-bd00b6639491 {"id":"5a15075d-43a4-47db-9ebb-bd00b6639491","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993923} /tidb/cdc/default/__cdc_meta__/capture/7a381120-6951-42dd-9332-33951b4de2ec {"id":"7a381120-6951-42dd-9332-33951b4de2ec","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993926} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a419bd2 5a15075d-43a4-47db-9ebb-bd00b6639491 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a419c08 7a381120-6951-42dd-9332-33951b4de2ec /tidb/cdc/default/default/upstream/7365842741345146855 {"id":7365842741345146855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a15075d-43a4-47db-9ebb-bd00b6639491 {"id":"5a15075d-43a4-47db-9ebb-bd00b6639491","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993923} /tidb/cdc/default/__cdc_meta__/capture/7a381120-6951-42dd-9332-33951b4de2ec {"id":"7a381120-6951-42dd-9332-33951b4de2ec","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993926} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a419bd2 5a15075d-43a4-47db-9ebb-bd00b6639491 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a419c08 7a381120-6951-42dd-9332-33951b4de2ec /tidb/cdc/default/default/upstream/7365842741345146855 {"id":7365842741345146855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a15075d-43a4-47db-9ebb-bd00b6639491 {"id":"5a15075d-43a4-47db-9ebb-bd00b6639491","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993923} /tidb/cdc/default/__cdc_meta__/capture/7a381120-6951-42dd-9332-33951b4de2ec {"id":"7a381120-6951-42dd-9332-33951b4de2ec","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993926} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a419bd2 5a15075d-43a4-47db-9ebb-bd00b6639491 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a419c08 7a381120-6951-42dd-9332-33951b4de2ec /tidb/cdc/default/default/upstream/7365842741345146855 + grep -q 'etcd info' {"id":7365842741345146855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Mon May 6 19:12:09 CST 2024] <<<<<< START cdc server in csv_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.1185111853.out server --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/multi_rocks Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... [Pipeline] } Verifying downstream PD is started... [Pipeline] // withEnv [Pipeline] } Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... check diff failed 2-th time, retry later [Pipeline] // stage cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Pipeline] } [Mon May 6 19:12:09 CST 2024] <<<<<< run test case ddl_attributes success! >>>>>> table split_region.test1 not exists for 4-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' 1 run task successfully check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:10 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2ba900b8-f1b1-4525-abc9-7534c93ae34a {"id":"2ba900b8-f1b1-4525-abc9-7534c93ae34a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993928} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a50d2d5 2ba900b8-f1b1-4525-abc9-7534c93ae34a /tidb/cdc/default/default/upstream/7365842765556795267 {"id":7365842765556795267,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2ba900b8-f1b1-4525-abc9-7534c93ae34a {"id":"2ba900b8-f1b1-4525-abc9-7534c93ae34a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993928} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a50d2d5 2ba900b8-f1b1-4525-abc9-7534c93ae34a /tidb/cdc/default/default/upstream/7365842765556795267 {"id":7365842765556795267,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2ba900b8-f1b1-4525-abc9-7534c93ae34a {"id":"2ba900b8-f1b1-4525-abc9-7534c93ae34a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993928} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a50d2d5 2ba900b8-f1b1-4525-abc9-7534c93ae34a /tidb/cdc/default/default/upstream/7365842765556795267 {"id":7365842765556795267,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5456.out cli changefeed create --start-ts=449575367796326401 '--sink-uri=file:///tmp/tidb_cdc_test/drop_many_tables/storage_test/ticdc-drop-tables-test-19173?protocol=canal-json&enable-tidb-extension=true' =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/common_1/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:12:10 CST 2024] <<<<<< run test case common_1 success! >>>>>> Create changefeed successfully! ID: d3d458ac-9f67-4375-a509-c5bc8d4c9dd6 Info: {"upstream_id":7365842765556795267,"namespace":"default","id":"d3d458ac-9f67-4375-a509-c5bc8d4c9dd6","sink_uri":"file:///tmp/tidb_cdc_test/drop_many_tables/storage_test/ticdc-drop-tables-test-19173?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:12:11.21464938+08:00","start_ts":449575367796326401,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575367796326401,"checkpoint_ts":449575367796326401,"checkpoint_time":"2024-05-06 19:12:06.225"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check diff failed 3-th time, retry later table split_region.test1 not exists for 5-th check, retry later Starting Upstream TiDB... start tidb cluster in /tmp/tidb_cdc_test/cli_with_auth Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:12 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b0daa43a-e64b-428d-a98a-ceed76c365d2 {"id":"b0daa43a-e64b-428d-a98a-ceed76c365d2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993929} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a492cf6 b0daa43a-e64b-428d-a98a-ceed76c365d2 /tidb/cdc/default/default/upstream/7365842764385118364 {"id":7365842764385118364,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b0daa43a-e64b-428d-a98a-ceed76c365d2 {"id":"b0daa43a-e64b-428d-a98a-ceed76c365d2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993929} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a492cf6 b0daa43a-e64b-428d-a98a-ceed76c365d2 /tidb/cdc/default/default/upstream/7365842764385118364 {"id":7365842764385118364,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b0daa43a-e64b-428d-a98a-ceed76c365d2 {"id":"b0daa43a-e64b-428d-a98a-ceed76c365d2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993929} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a492cf6 b0daa43a-e64b-428d-a98a-ceed76c365d2 /tidb/cdc/default/default/upstream/7365842764385118364 {"id":7365842764385118364,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5312.out cli changefeed create --start-ts=449575367979040769 '--sink-uri=file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-3766?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:12 GMT < Content-Length: 1756 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a15075d-43a4-47db-9ebb-bd00b6639491 {"id":"5a15075d-43a4-47db-9ebb-bd00b6639491","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993923} /tidb/cdc/default/__cdc_meta__/capture/7a381120-6951-42dd-9332-33951b4de2ec {"id":"7a381120-6951-42dd-9332-33951b4de2ec","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993926} /tidb/cdc/default/__cdc_meta__/capture/bc7f01f4-4312-4146-9591-5a688e557177 {"id":"bc7f01f4-4312-4146-9591-5a688e557177","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993929} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a419bd2 5a15075d-43a4-47db-9ebb-bd00b6639491 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a419c08 7a381120-6951-42dd-9332-33951b4de2ec /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a419c27 bc7f01f4-4312-4146-9591-5a688e557177 /tidb/cdc/default/default/upstream/7365842741345146855 {"id":7365842741345146855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a15075d-43a4-47db-9ebb-bd00b6639491 {"id":"5a15075d-43a4-47db-9ebb-bd00b6639491","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993923} /tidb/cdc/default/__cdc_meta__/capture/7a381120-6951-42dd-9332-33951b4de2ec {"id":"7a381120-6951-42dd-9332-33951b4de2ec","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993926} /tidb/cdc/default/__cdc_meta__/capture/bc7f01f4-4312-4146-9591-5a688e557177 {"id":"bc7f01f4-4312-4146-9591-5a688e557177","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993929} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a419bd2 5a15075d-43a4-47db-9ebb-bd00b6639491 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a419c08 7a381120-6951-42dd-9332-33951b4de2ec /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a419c27 bc7f01f4-4312-4146-9591-5a688e557177 /tidb/cdc/default/default/upstream/7365842741345146855 {"id":7365842741345146855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a15075d-43a4-47db-9ebb-bd00b6639491 {"id":"5a15075d-43a4-47db-9ebb-bd00b6639491","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993923} /tidb/cdc/default/__cdc_meta__/capture/7a381120-6951-42dd-9332-33951b4de2ec {"id":"7a381120-6951-42dd-9332-33951b4de2ec","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993926} /tidb/cdc/default/__cdc_meta__/capture/bc7f01f4-4312-4146-9591-5a688e557177 {"id":"bc7f01f4-4312-4146-9591-5a688e557177","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993929} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a419bd2 5a15075d-43a4-47db-9ebb-bd00b6639491 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a419c08 7a381120-6951-42dd-9332-33951b4de2ec /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a419c27 bc7f01f4-4312-4146-9591-5a688e557177 /tidb/cdc/default/default/upstream/7365842741345146855 {"id":7365842741345146855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.cli.11897.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml check diff failed 2-th time, retry later + set +x + workdir=/tmp/tidb_cdc_test/drop_many_tables + sink_uri='file:///tmp/tidb_cdc_test/drop_many_tables/storage_test/ticdc-drop-tables-test-19173?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/drop_many_tables ++ date + echo '[Mon May 6 19:12:12 CST 2024] <<<<<< START storage consumer in drop_many_tables case >>>>>>' [Mon May 6 19:12:12 CST 2024] <<<<<< START storage consumer in drop_many_tables case >>>>>> + cd /tmp/tidb_cdc_test/drop_many_tables + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/drop_many_tables + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/drop_many_tables/storage_test/ticdc-drop-tables-test-19173?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7365842764385118364,"namespace":"default","id":"changefeed-error","sink_uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-3766?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:12:12.700959804+08:00","start_ts":449575367979040769,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575367979040769,"checkpoint_ts":449575367979040769,"checkpoint_time":"2024-05-06 19:12:06.922"} PASS Create changefeed successfully! ID: 0c30e564-fe7f-47d2-b18c-f8df0231bc19 Info: {"upstream_id":7365842741345146855,"namespace":"default","id":"0c30e564-fe7f-47d2-b18c-f8df0231bc19","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:12:12.729542023+08:00","start_ts":449575369476145157,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575369476145157,"checkpoint_ts":449575369476145157,"checkpoint_time":"2024-05-06 19:12:12.633"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used coverage: 2.5% of statements in github.com/pingcap/tiflow/... table drop_tables.c not exists for 1-th check, retry later check diff failed 4-th time, retry later table split_region.test1 not exists for 6-th check, retry later + set +x + set +x + workdir=/tmp/tidb_cdc_test/changefeed_error + sink_uri='file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-3766?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/changefeed_error ++ date + echo '[Mon May 6 19:12:14 CST 2024] <<<<<< START storage consumer in changefeed_error case >>>>>>' [Mon May 6 19:12:14 CST 2024] <<<<<< START storage consumer in changefeed_error case >>>>>> + cd /tmp/tidb_cdc_test/changefeed_error + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/changefeed_error + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/changefeed_error/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-3766?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later + info='{ "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449575367979040769, "checkpoint_time": "2024-05-06 19:12:06.922", "error": { "time": "2024-05-06T19:12:12.841597324+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449575367979040469 is earlier than or equal to GC safepoint at 449575367979040769" } }' + echo '{ "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449575367979040769, "checkpoint_time": "2024-05-06 19:12:06.922", "error": { "time": "2024-05-06T19:12:12.841597324+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449575367979040469 is earlier than or equal to GC safepoint at 449575367979040769" } }' { "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449575367979040769, "checkpoint_time": "2024-05-06 19:12:06.922", "error": { "time": "2024-05-06T19:12:12.841597324+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449575367979040469 is earlier than or equal to GC safepoint at 449575367979040769" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365842764385118364, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449575367979040769, '"checkpoint_time":' '"2024-05-06' '19:12:06.922",' '"error":' '{' '"time":' '"2024-05-06T19:12:12.841597324+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449575367979040469 is earlier than or equal to GC safepoint at '449575367979040769"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365842764385118364, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449575367979040769, '"checkpoint_time":' '"2024-05-06' '19:12:06.922",' '"error":' '{' '"time":' '"2024-05-06T19:12:12.841597324+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449575367979040469 is earlier than or equal to GC safepoint at '449575367979040769"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449575367979040469 is earlier than or equal to GC safepoint at 449575367979040769' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449575367979040469 is earlier than or equal to GC safepoint at 449575367979040769 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5418.out cli changefeed resume -c changefeed-error Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/event_filter/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... coverage: 2.1% of statements in github.com/pingcap/tiflow/... check diff failed 5-th time, retry later table drop_tables.c not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff successfully table split_region.test1 exists table split_region.test2 exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table changefeed_error.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... table drop_tables.c not exists for 3-th check, retry later wait process cdc.test exit for 3-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 6-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:12:18 CST 2024] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_error.usertable not exists for 2-th check, retry later table drop_tables.c not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366af4bc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:7246, start at 2024-05-06 19:12:17.734963652 +0800 CST m=+5.395224917 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:17.741 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:17.711 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:17.711 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366af4bc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:7246, start at 2024-05-06 19:12:17.734963652 +0800 CST m=+5.395224917 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:17.741 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:17.711 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:17.711 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366af6280005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:7330, start at 2024-05-06 19:12:17.805734181 +0800 CST m=+5.385354994 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:17.814 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:17.802 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:17.802 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table changefeed_error.usertable not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc already exists) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } table drop_tables.c not exists for 5-th check, retry later [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // node [Pipeline] } \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] // podTemplate [Pipeline] } [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc already exists) [Pipeline] // withEnv [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // dir [Pipeline] } [Pipeline] } Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // stage table changefeed_error.usertable not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8642.out cli tso query --pd=http://127.0.0.1:2379 table drop_tables.c not exists for 6-th check, retry later [Pipeline] // container [Pipeline] } [Pipeline] // withEnv + set +x + tso='449575372196937729 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575372196937729 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_error.usertable exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "recordcount"="20" "mysql.user"="root" "mysql.db"="changefeed_error" "dotransactions"="false" "readallfields"="true" "mysql.host"="127.0.0.1" "updateproportion"="0" "requestdistribution"="uniform" "operationcount"="0" "readproportion"="0" "mysql.port"="4000" "threadcount"="4" "workload"="core" "insertproportion"="0" "scanproportion"="0" ********************************************** Run finished, takes 4.67356ms INSERT - Takes(s): 0.0, Count: 20, OPS: 6219.4, Avg(us): 793, Min(us): 456, Max(us): 1860, 95th(us): 2000, 99th(us): 2000 check diff successfully {"id":"b0daa43a-e64b-428d-a98a-ceed76c365d2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993929} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/b0daa43a-e64b-428d-a98a-ceed76c365d2 + [[ /tidb/cdc/default/__cdc_meta__/capture/b0daa43a-e64b-428d-a98a-ceed76c365d2 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/b0daa43a-e64b-428d-a98a-ceed76c365d2' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/b0daa43a-e64b-428d-a98a-ceed76c365d2 + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage ***************** properties ***************** "mysql.db"="multi_capture_1" "insertproportion"="0" "updateproportion"="0" "workload"="core" "requestdistribution"="uniform" "mysql.user"="root" "readproportion"="0" "mysql.host"="127.0.0.1" "operationcount"="0" "scanproportion"="0" "readallfields"="true" "dotransactions"="false" "recordcount"="10" "threadcount"="2" "mysql.port"="4000" ********************************************** Run finished, takes 9.839578ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1939.2, Avg(us): 1891, Min(us): 1134, Max(us): 4591, 95th(us): 5000, 99th(us): 5000 [Pipeline] } table drop_tables.c not exists for 7-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366b503c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:11060, start at 2024-05-06 19:12:23.573169263 +0800 CST m=+5.589885456 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:23.580 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:23.567 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:23.567 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366b503c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:11060, start at 2024-05-06 19:12:23.573169263 +0800 CST m=+5.589885456 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:23.580 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:23.567 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:23.567 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366b51980005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:11126, start at 2024-05-06 19:12:23.660635757 +0800 CST m=+5.611471184 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:23.668 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:23.654 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:23.654 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ***************** properties ***************** "updateproportion"="0" "recordcount"="10" "requestdistribution"="uniform" "mysql.user"="root" "dotransactions"="false" "mysql.port"="4000" "mysql.host"="127.0.0.1" "workload"="core" "readproportion"="0" "operationcount"="0" "insertproportion"="0" "mysql.db"="multi_capture_2" "threadcount"="2" "readallfields"="true" "scanproportion"="0" ********************************************** Run finished, takes 10.670124ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1839.0, Avg(us): 2045, Min(us): 1039, Max(us): 5128, 95th(us): 6000, 99th(us): 6000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/b0daa43a-e64b-428d-a98a-ceed76c365d2 + [[ /tidb/cdc/default/__cdc_meta__/capture/b0daa43a-e64b-428d-a98a-ceed76c365d2 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/b0daa43a-e64b-428d-a98a-ceed76c365d2' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/b0daa43a-e64b-428d-a98a-ceed76c365d2 + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "readproportion"="0" "workload"="core" "scanproportion"="0" "requestdistribution"="uniform" "recordcount"="10" "mysql.db"="multi_capture_3" "threadcount"="2" "dotransactions"="false" "readallfields"="true" "updateproportion"="0" "mysql.host"="127.0.0.1" "insertproportion"="0" "mysql.port"="4000" "mysql.user"="root" "operationcount"="0" ********************************************** Run finished, takes 9.835716ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1866.5, Avg(us): 1888, Min(us): 1163, Max(us): 4459, 95th(us): 5000, 99th(us): 5000 table drop_tables.c not exists for 8-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12366.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later ***************** properties ***************** "mysql.host"="127.0.0.1" "mysql.db"="multi_capture_4" "operationcount"="0" "requestdistribution"="uniform" "recordcount"="10" "readproportion"="0" "threadcount"="2" "workload"="core" "mysql.user"="root" "scanproportion"="0" "readallfields"="true" "updateproportion"="0" "insertproportion"="0" "dotransactions"="false" "mysql.port"="4000" ********************************************** Run finished, takes 9.981361ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1710.3, Avg(us): 1921, Min(us): 1201, Max(us): 4091, 95th(us): 5000, 99th(us): 5000 [Mon May 6 19:12:28 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.87758777.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table drop_tables.c not exists for 9-th check, retry later + set +x + tso='449575373679362050 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575373679362050 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x check diff successfully check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc already exists) [Pipeline] // cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366b96c80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:6459, start at 2024-05-06 19:12:28.08737883 +0800 CST m=+5.357542575 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:28.094 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:28.082 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:28.082 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366b96c80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:6459, start at 2024-05-06 19:12:28.08737883 +0800 CST m=+5.357542575 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:28.094 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:28.082 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:28.082 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366b96b4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:6540, start at 2024-05-06 19:12:28.091640694 +0800 CST m=+5.314578732 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:28.097 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:28.077 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:28.077 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/event_filter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/event_filter/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] } check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/b0daa43a-e64b-428d-a98a-ceed76c365d2 + [[ /tidb/cdc/default/__cdc_meta__/capture/b0daa43a-e64b-428d-a98a-ceed76c365d2 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/b0daa43a-e64b-428d-a98a-ceed76c365d2' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/b0daa43a-e64b-428d-a98a-ceed76c365d2 + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // stage [Pipeline] } [Mon May 6 19:12:31 CST 2024] <<<<<< START cdc server in cli_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.1241212414.out server --log-file /tmp/tidb_cdc_test/cli_with_auth/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_with_auth/cdc_data --cluster-id default --config /tmp/tidb_cdc_test/cli_with_auth/server.toml ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } wait process cdc.test exit for 3-th time... [Pipeline] // stage [Pipeline] } table drop_tables.c not exists for 10-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:12:32 CST 2024] <<<<<< run test case owner_resign success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:31 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e3e6208-21ad-423f-a554-b596194cb274 {"id":"8e3e6208-21ad-423f-a554-b596194cb274","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993949} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a961b50 8e3e6208-21ad-423f-a554-b596194cb274 /tidb/cdc/default/default/upstream/7365842837485107141 {"id":7365842837485107141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e3e6208-21ad-423f-a554-b596194cb274 {"id":"8e3e6208-21ad-423f-a554-b596194cb274","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993949} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a961b50 8e3e6208-21ad-423f-a554-b596194cb274 /tidb/cdc/default/default/upstream/7365842837485107141 {"id":7365842837485107141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e3e6208-21ad-423f-a554-b596194cb274 {"id":"8e3e6208-21ad-423f-a554-b596194cb274","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993949} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a961b50 8e3e6208-21ad-423f-a554-b596194cb274 /tidb/cdc/default/default/upstream/7365842837485107141 {"id":7365842837485107141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:12:31 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.88318833.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 19:12:33 CST 2024] <<<<<< START cdc server in event_filter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.79187920.out server --log-file /tmp/tidb_cdc_test/event_filter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/event_filter/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table drop_tables.c exists check diff successfully wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:34 GMT < Content-Length: 861 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f9df963-aac3-4e0c-82d8-7bdf34c6809b {"id":"6f9df963-aac3-4e0c-82d8-7bdf34c6809b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993951} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9aae209d 6f9df963-aac3-4e0c-82d8-7bdf34c6809b /tidb/cdc/default/default/upstream/7365842861521103719 {"id":7365842861521103719,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f9df963-aac3-4e0c-82d8-7bdf34c6809b {"id":"6f9df963-aac3-4e0c-82d8-7bdf34c6809b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993951} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9aae209d 6f9df963-aac3-4e0c-82d8-7bdf34c6809b /tidb/cdc/default/default/upstream/7365842861521103719 {"id":7365842861521103719,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f9df963-aac3-4e0c-82d8-7bdf34c6809b {"id":"6f9df963-aac3-4e0c-82d8-7bdf34c6809b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993951} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9aae209d 6f9df963-aac3-4e0c-82d8-7bdf34c6809b /tidb/cdc/default/default/upstream/7365842861521103719 {"id":7365842861521103719,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12468.out cli changefeed create --start-ts=449575373679362050 '--sink-uri=file:///tmp/tidb_cdc_test/cli_with_auth/storage_test/ticdc-cli-test-11714?protocol=canal-json&enable-tidb-extension=true' --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. wait process cdc.test exit for 2-th time... Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365842861521103719,"namespace":"default","id":"custom-changefeed-name","sink_uri":"file:///tmp/tidb_cdc_test/cli_with_auth/storage_test/ticdc-cli-test-11714?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:12:34.867110854+08:00","start_ts":449575373679362050,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575373679362050,"checkpoint_ts":449575373679362050,"checkpoint_time":"2024-05-06 19:12:28.667"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:12:35 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:34 GMT < Content-Length: 1275 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b625087-f83b-434e-8863-cb975f4fc4c8 {"id":"8b625087-f83b-434e-8863-cb975f4fc4c8","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993952} /tidb/cdc/default/__cdc_meta__/capture/8e3e6208-21ad-423f-a554-b596194cb274 {"id":"8e3e6208-21ad-423f-a554-b596194cb274","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993949} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a961b50 8e3e6208-21ad-423f-a554-b596194cb274 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a961b8a 8b625087-f83b-434e-8863-cb975f4fc4c8 /tidb/cdc/default/default/upstream/7365842837485107141 {"id":7365842837485107141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b625087-f83b-434e-8863-cb975f4fc4c8 {"id":"8b625087-f83b-434e-8863-cb975f4fc4c8","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993952} /tidb/cdc/default/__cdc_meta__/capture/8e3e6208-21ad-423f-a554-b596194cb274 {"id":"8e3e6208-21ad-423f-a554-b596194cb274","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993949} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a961b50 8e3e6208-21ad-423f-a554-b596194cb274 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a961b8a 8b625087-f83b-434e-8863-cb975f4fc4c8 /tidb/cdc/default/default/upstream/7365842837485107141 {"id":7365842837485107141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b625087-f83b-434e-8863-cb975f4fc4c8 {"id":"8b625087-f83b-434e-8863-cb975f4fc4c8","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993952} /tidb/cdc/default/__cdc_meta__/capture/8e3e6208-21ad-423f-a554-b596194cb274 {"id":"8e3e6208-21ad-423f-a554-b596194cb274","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993949} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a961b50 8e3e6208-21ad-423f-a554-b596194cb274 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a961b8a 8b625087-f83b-434e-8863-cb975f4fc4c8 /tidb/cdc/default/default/upstream/7365842837485107141 {"id":7365842837485107141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:12:34 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.88858887.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:36 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c1b5199-9c69-4034-804e-7c485fb8722f {"id":"7c1b5199-9c69-4034-804e-7c485fb8722f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993953} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9abae6c9 7c1b5199-9c69-4034-804e-7c485fb8722f /tidb/cdc/default/default/upstream/7365842883396293961 {"id":7365842883396293961,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c1b5199-9c69-4034-804e-7c485fb8722f {"id":"7c1b5199-9c69-4034-804e-7c485fb8722f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993953} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9abae6c9 7c1b5199-9c69-4034-804e-7c485fb8722f /tidb/cdc/default/default/upstream/7365842883396293961 {"id":7365842883396293961,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c1b5199-9c69-4034-804e-7c485fb8722f {"id":"7c1b5199-9c69-4034-804e-7c485fb8722f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993953} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9abae6c9 7c1b5199-9c69-4034-804e-7c485fb8722f /tidb/cdc/default/default/upstream/7365842883396293961 {"id":7365842883396293961,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.cli.7982.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/event_filter/storage_test/ticdc-event-filter-23644?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/event_filter/conf/cf.toml wait process cdc.test exit for 3-th time... + set +x + workdir=/tmp/tidb_cdc_test/cli_with_auth + sink_uri='file:///tmp/tidb_cdc_test/cli_with_auth/storage_test/ticdc-cli-test-11714?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/cli_with_auth ++ date + echo '[Mon May 6 19:12:36 CST 2024] <<<<<< START storage consumer in cli_with_auth case >>>>>>' [Mon May 6 19:12:36 CST 2024] <<<<<< START storage consumer in cli_with_auth case >>>>>> + cd /tmp/tidb_cdc_test/cli_with_auth + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/cli_with_auth + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/cli_with_auth/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/cli_with_auth/storage_test/ticdc-cli-test-11714?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table test.simple not exists for 1-th check, retry later Create changefeed successfully! ID: e4da5f38-a507-4ec7-b8b2-b38396b1cd67 Info: {"upstream_id":7365842883396293961,"namespace":"default","id":"e4da5f38-a507-4ec7-b8b2-b38396b1cd67","sink_uri":"file:///tmp/tidb_cdc_test/event_filter/storage_test/ticdc-event-filter-23644?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:12:36.569954509+08:00","start_ts":449575375728279555,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["event_filter.*"],"event_filters":[{"matcher":["event_filter.t1"],"ignore_event":["drop table","delete"],"ignore_sql":null,"ignore_insert_value_expr":"id = 2 or city = 'tokyo'","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_truncate"],"ignore_event":["truncate table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_alter"],"ignore_event":["alter table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""}]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575375728279555,"checkpoint_ts":449575375728279555,"checkpoint_time":"2024-05-06 19:12:36.483"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:12:36 CST 2024] <<<<<< run test case split_region success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 19:12:37 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.56435645.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + workdir=/tmp/tidb_cdc_test/event_filter + sink_uri='file:///tmp/tidb_cdc_test/event_filter/storage_test/ticdc-event-filter-23644?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/event_filter ++ date + echo '[Mon May 6 19:12:38 CST 2024] <<<<<< START storage consumer in event_filter case >>>>>>' [Mon May 6 19:12:38 CST 2024] <<<<<< START storage consumer in event_filter case >>>>>> + cd /tmp/tidb_cdc_test/event_filter + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/event_filter + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/event_filter/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/event_filter/storage_test/ticdc-event-filter-23644?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:37 GMT < Content-Length: 1756 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b625087-f83b-434e-8863-cb975f4fc4c8 {"id":"8b625087-f83b-434e-8863-cb975f4fc4c8","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993952} /tidb/cdc/default/__cdc_meta__/capture/8e3e6208-21ad-423f-a554-b596194cb274 {"id":"8e3e6208-21ad-423f-a554-b596194cb274","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993949} /tidb/cdc/default/__cdc_meta__/capture/a6d6ae09-3d6e-437b-b881-a1a862158ab6 {"id":"a6d6ae09-3d6e-437b-b881-a1a862158ab6","address":"127.0.0.1:8303","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993955} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a961b50 8e3e6208-21ad-423f-a554-b596194cb274 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a961b8a 8b625087-f83b-434e-8863-cb975f4fc4c8 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a961baa a6d6ae09-3d6e-437b-b881-a1a862158ab6 /tidb/cdc/default/default/upstream/7365842837485107141 {"id":7365842837485107141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b625087-f83b-434e-8863-cb975f4fc4c8 {"id":"8b625087-f83b-434e-8863-cb975f4fc4c8","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993952} /tidb/cdc/default/__cdc_meta__/capture/8e3e6208-21ad-423f-a554-b596194cb274 {"id":"8e3e6208-21ad-423f-a554-b596194cb274","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993949} /tidb/cdc/default/__cdc_meta__/capture/a6d6ae09-3d6e-437b-b881-a1a862158ab6 {"id":"a6d6ae09-3d6e-437b-b881-a1a862158ab6","address":"127.0.0.1:8303","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993955} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a961b50 8e3e6208-21ad-423f-a554-b596194cb274 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a961b8a 8b625087-f83b-434e-8863-cb975f4fc4c8 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a961baa a6d6ae09-3d6e-437b-b881-a1a862158ab6 /tidb/cdc/default/default/upstream/7365842837485107141 {"id":7365842837485107141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b625087-f83b-434e-8863-cb975f4fc4c8 {"id":"8b625087-f83b-434e-8863-cb975f4fc4c8","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993952} /tidb/cdc/default/__cdc_meta__/capture/8e3e6208-21ad-423f-a554-b596194cb274 {"id":"8e3e6208-21ad-423f-a554-b596194cb274","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993949} /tidb/cdc/default/__cdc_meta__/capture/a6d6ae09-3d6e-437b-b881-a1a862158ab6 {"id":"a6d6ae09-3d6e-437b-b881-a1a862158ab6","address":"127.0.0.1:8303","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993955} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a961b50 8e3e6208-21ad-423f-a554-b596194cb274 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a961b8a 8b625087-f83b-434e-8863-cb975f4fc4c8 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a961baa a6d6ae09-3d6e-437b-b881-a1a862158ab6 /tidb/cdc/default/default/upstream/7365842837485107141 {"id":7365842837485107141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8941.out cli changefeed create --start-ts=449575372196937729 '--sink-uri=file:///tmp/tidb_cdc_test/multi_capture/storage_test/ticdc-multi-capture-test-8777?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 Create changefeed successfully! ID: 0b541995-4b6a-44ce-bb7e-b7321f3d0438 Info: {"upstream_id":7365842837485107141,"namespace":"default","id":"0b541995-4b6a-44ce-bb7e-b7321f3d0438","sink_uri":"file:///tmp/tidb_cdc_test/multi_capture/storage_test/ticdc-multi-capture-test-8777?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:12:38.407649672+08:00","start_ts":449575372196937729,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575372196937729,"checkpoint_ts":449575372196937729,"checkpoint_time":"2024-05-06 19:12:23.012"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.simple not exists for 2-th check, retry later + set +x + workdir=/tmp/tidb_cdc_test/multi_capture + sink_uri='file:///tmp/tidb_cdc_test/multi_capture/storage_test/ticdc-multi-capture-test-8777?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/multi_capture ++ date + echo '[Mon May 6 19:12:39 CST 2024] <<<<<< START storage consumer in multi_capture case >>>>>>' [Mon May 6 19:12:39 CST 2024] <<<<<< START storage consumer in multi_capture case >>>>>> + cd /tmp/tidb_cdc_test/multi_capture + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/multi_capture + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/multi_capture/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/multi_capture/storage_test/ticdc-multi-capture-test-8777?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table multi_capture_1.usertable not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:40 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/be8b64de-7fe2-4dc6-83dc-b624b8a86d80 {"id":"be8b64de-7fe2-4dc6-83dc-b624b8a86d80","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993957} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a492e02 be8b64de-7fe2-4dc6-83dc-b624b8a86d80 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365842764385118364,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-3766?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:12:12.700959804+08:00","start-ts":449575367979040769,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-06T19:12:37.724569586+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575376053075970} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449575372514394116,"min-table-barrier-ts":449575372514394116,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365842764385118364 {"id":7365842764385118364,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/be8b64de-7fe2-4dc6-83dc-b624b8a86d80 {"id":"be8b64de-7fe2-4dc6-83dc-b624b8a86d80","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993957} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a492e02 be8b64de-7fe2-4dc6-83dc-b624b8a86d80 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365842764385118364,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-3766?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:12:12.700959804+08:00","start-ts":449575367979040769,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-06T19:12:37.724569586+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575376053075970} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449575372514394116,"min-table-barrier-ts":449575372514394116,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365842764385118364 {"id":7365842764385118364,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/be8b64de-7fe2-4dc6-83dc-b624b8a86d80 {"id":"be8b64de-7fe2-4dc6-83dc-b624b8a86d80","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993957} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a492e02 be8b64de-7fe2-4dc6-83dc-b624b8a86d80 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365842764385118364,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-3766?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:12:12.700959804+08:00","start-ts":449575367979040769,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-06T19:12:37.724569586+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575376053075970} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449575372514394116,"min-table-barrier-ts":449575372514394116,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365842764385118364 {"id":7365842764385118364,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449575372514394116, "checkpoint_time": "2024-05-06 19:12:24.223", "error": { "time": "2024-05-06T19:12:37.724569586+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449575372514394116, "checkpoint_time": "2024-05-06 19:12:24.223", "error": { "time": "2024-05-06T19:12:37.724569586+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449575372514394116, "checkpoint_time": "2024-05-06 19:12:24.223", "error": { "time": "2024-05-06T19:12:37.724569586+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7365842764385118364, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449575372514394116, '"checkpoint_time":' '"2024-05-06' '19:12:24.223",' '"error":' '{' '"time":' '"2024-05-06T19:12:37.724569586+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7365842764385118364, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449575372514394116, '"checkpoint_time":' '"2024-05-06' '19:12:24.223",' '"error":' '{' '"time":' '"2024-05-06T19:12:37.724569586+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + workdir=/tmp/tidb_cdc_test/csv_storage_partition_table + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_partition_table ++ date + echo '[Mon May 6 19:12:38 CST 2024] <<<<<< START storage consumer in csv_storage_partition_table case >>>>>>' [Mon May 6 19:12:38 CST 2024] <<<<<< START storage consumer in csv_storage_partition_table case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_partition_table + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_partition_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5745.out cli changefeed remove -c changefeed-error ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.simple not exists for 3-th check, retry later table event_filter.t1 does not exists table event_filter.t1 not exists for 1-th check, retry later Changefeed remove successfully. ID: changefeed-error CheckpointTs: 449575372514394116 SinkURI: file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-3766?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table multi_capture_1.usertable not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366c6c8c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-4h8-1vqcr, pid:3959, start at 2024-05-06 19:12:41.79296968 +0800 CST m=+27.239767728 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:41.799 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:41.763 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:41.763 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366c6c8c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-4h8-1vqcr, pid:3959, start at 2024-05-06 19:12:41.79296968 +0800 CST m=+27.239767728 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:41.799 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:41.763 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:41.763 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366b36680013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-4h8-1vqcr, pid:4044, start at 2024-05-06 19:12:21.938380729 +0800 CST m=+7.334619325 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:21.949 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:21.914 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:21.914 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table test.simple not exists for 4-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully table event_filter.t1 not exists for 2-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only table multi_capture_1.usertable not exists for 3-th check, retry later table test.simple not exists for 5-th check, retry later + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 19:12:44 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.58365838.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 19:12:44 CST 2024] <<<<<< START cdc server in multi_rocks case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.1674916751.out server --log-file /tmp/tidb_cdc_test/multi_rocks/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_rocks/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table event_filter.t1 not exists for 3-th check, retry later table multi_capture_1.usertable not exists for 4-th check, retry later table test.simple not exists for 6-th check, retry later table partition_table.t not exists for 1-th check, retry later table event_filter.t1 not exists for 4-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:47 GMT < Content-Length: 885 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fb4a4119-bd1d-4150-a8e7-ec5e1af95e45 {"id":"fb4a4119-bd1d-4150-a8e7-ec5e1af95e45","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993964} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a492e5c fb4a4119-bd1d-4150-a8e7-ec5e1af95e45 /tidb/cdc/default/default/upstream/7365842764385118364 {"id":7365842764385118364,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fb4a4119-bd1d-4150-a8e7-ec5e1af95e45 {"id":"fb4a4119-bd1d-4150-a8e7-ec5e1af95e45","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993964} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a492e5c fb4a4119-bd1d-4150-a8e7-ec5e1af95e45 /tidb/cdc/default/default/upstream/7365842764385118364 {"id":7365842764385118364,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fb4a4119-bd1d-4150-a8e7-ec5e1af95e45 {"id":"fb4a4119-bd1d-4150-a8e7-ec5e1af95e45","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993964} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a492e5c fb4a4119-bd1d-4150-a8e7-ec5e1af95e45 /tidb/cdc/default/default/upstream/7365842764385118364 {"id":7365842764385118364,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5895.out cli changefeed create --start-ts=449575367979040769 '--sink-uri=file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-3766?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error-1 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7365842764385118364,"namespace":"default","id":"changefeed-error-1","sink_uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-3766?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:12:47.964217755+08:00","start_ts":449575367979040769,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575367979040769,"checkpoint_ts":449575367979040769,"checkpoint_time":"2024-05-06 19:12:06.922"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table multi_capture_1.usertable not exists for 5-th check, retry later < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:48 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/37d7a2e5-9af0-4f8c-a132-6e9d87db4d26 {"id":"37d7a2e5-9af0-4f8c-a132-6e9d87db4d26","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993965} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a9cf200 37d7a2e5-9af0-4f8c-a132-6e9d87db4d26 /tidb/cdc/default/default/upstream/7365842850778304244 {"id":7365842850778304244,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/37d7a2e5-9af0-4f8c-a132-6e9d87db4d26 {"id":"37d7a2e5-9af0-4f8c-a132-6e9d87db4d26","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993965} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a9cf200 37d7a2e5-9af0-4f8c-a132-6e9d87db4d26 /tidb/cdc/default/default/upstream/7365842850778304244 {"id":7365842850778304244,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/37d7a2e5-9af0-4f8c-a132-6e9d87db4d26 {"id":"37d7a2e5-9af0-4f8c-a132-6e9d87db4d26","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993965} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a9cf200 37d7a2e5-9af0-4f8c-a132-6e9d87db4d26 /tidb/cdc/default/default/upstream/7365842850778304244 {"id":7365842850778304244,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.cli.17349.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/multi_rocks/storage_test/ticdc-multi-rocks-test-11878?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: 6977c05b-ccb1-40b5-9ba3-f49ff87d279c Info: {"upstream_id":7365842850778304244,"namespace":"default","id":"6977c05b-ccb1-40b5-9ba3-f49ff87d279c","sink_uri":"file:///tmp/tidb_cdc_test/multi_rocks/storage_test/ticdc-multi-rocks-test-11878?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:12:48.460037399+08:00","start_ts":449575378842550275,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575378842550275,"checkpoint_ts":449575378842550275,"checkpoint_time":"2024-05-06 19:12:48.363"} PASS table partition_table.t not exists for 2-th check, retry later coverage: 2.5% of statements in github.com/pingcap/tiflow/... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc already exists) table test.simple exists table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365842861521103719, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575378764955654, "checkpoint_time": "2024-05-06 19:12:48.067", "error": null }' + echo '{ "upstream_id": 7365842861521103719, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575378764955654, "checkpoint_time": "2024-05-06 19:12:48.067", "error": null }' { "upstream_id": 7365842861521103719, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575378764955654, "checkpoint_time": "2024-05-06 19:12:48.067", "error": null } ++ echo '{' '"upstream_id":' 7365842861521103719, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575378764955654, '"checkpoint_time":' '"2024-05-06' '19:12:48.067",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365842861521103719, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575378764955654, '"checkpoint_time":' '"2024-05-06' '19:12:48.067",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } table event_filter.t1 exists table event_filter.t_normal exists table event_filter.t_truncate not exists for 1-th check, retry later changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 [Pipeline] // container + set +x [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 253 100 253 0 0 1829 0 --:--:-- --:--:-- --:--:-- 1833 + info='{"state":"warning","resolved_ts":449575368490221569,"checkpoint_ts":449575368490221569,"last_warning":{"time":"2024-05-06T19:12:49.455099166+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":449575368490221569,"checkpoint_ts":449575368490221569,"last_warning":{"time":"2024-05-06T19:12:49.455099166+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":449575368490221569,"checkpoint_ts":449575368490221569,"last_warning":{"time":"2024-05-06T19:12:49.455099166+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ echo '{"state":"warning","resolved_ts":449575368490221569,"checkpoint_ts":449575368490221569,"last_warning":{"time":"2024-05-06T19:12:49.455099166+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ jq -r .last_warning.message ++ echo '{"state":"warning","resolved_ts":449575368490221569,"checkpoint_ts":449575368490221569,"last_warning":{"time":"2024-05-06T19:12:49.455099166+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5944.out cli changefeed remove -c changefeed-error-1 [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table multi_capture_1.usertable not exists for 6-th check, retry later + set +x + workdir=/tmp/tidb_cdc_test/multi_rocks + sink_uri='file:///tmp/tidb_cdc_test/multi_rocks/storage_test/ticdc-multi-rocks-test-11878?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/multi_rocks ++ date + echo '[Mon May 6 19:12:49 CST 2024] <<<<<< START storage consumer in multi_rocks case >>>>>>' [Mon May 6 19:12:49 CST 2024] <<<<<< START storage consumer in multi_rocks case >>>>>> + cd /tmp/tidb_cdc_test/multi_rocks + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/multi_rocks + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/multi_rocks/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/multi_rocks/storage_test/ticdc-multi-rocks-test-11878?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc0013c0bd0} {Type:update Path:[Config SyncPointRetention] From: To:0xc0013c0bd8} {Type:update Path:[Config Consistent] From: To:0xc00192a2a0} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12857.out cli changefeed --changefeed-id custom-changefeed-name pause Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 449575368490221569 SinkURI: file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-3766?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... table partition_table.t exists table partition_table.t1 exists table partition_table.finish_mark not exists for 1-th check, retry later ***************** properties ***************** "readproportion"="0" "requestdistribution"="uniform" "workload"="core" "mysql.port"="4000" "mysql.user"="root" "updateproportion"="0" "recordcount"="1000" "dotransactions"="false" "scanproportion"="0" "operationcount"="0" "table"="a1" "threadcount"="2" "mysql.host"="127.0.0.1" "insertproportion"="0" "mysql.db"="multi_rocks" "readallfields"="true" ********************************************** start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table event_filter.t_truncate not exists for 2-th check, retry later Run finished, takes 611.1603ms INSERT - Takes(s): 0.6, Count: 1000, OPS: 1669.4, Avg(us): 1188, Min(us): 889, Max(us): 15816, 95th(us): 2000, 99th(us): 2000 table multi_capture_1.usertable exists table multi_capture_2.usertable exists table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later + set +x ***************** properties ***************** "workload"="core" "mysql.db"="multi_rocks" "mysql.port"="4000" "dotransactions"="false" "insertproportion"="0" "mysql.user"="root" "threadcount"="2" "scanproportion"="0" "table"="a2" "recordcount"="1000" "updateproportion"="0" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "readproportion"="0" "readallfields"="true" "operationcount"="0" ********************************************** + set +x wait process cdc.test exit for 1-th time... Run finished, takes 560.965832ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1923.2, Avg(us): 1089, Min(us): 813, Max(us): 40918, 95th(us): 2000, 99th(us): 2000 start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table partition_table.finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... table event_filter.t_truncate not exists for 3-th check, retry later ***************** properties ***************** "readproportion"="0" "recordcount"="1000" "mysql.user"="root" "operationcount"="0" "readallfields"="true" "scanproportion"="0" "requestdistribution"="uniform" "updateproportion"="0" "mysql.host"="127.0.0.1" "mysql.db"="multi_rocks" "dotransactions"="false" "workload"="core" "mysql.port"="4000" "table"="a3" "insertproportion"="0" "threadcount"="2" ********************************************** cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 19:12:53 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.60086010.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Run finished, takes 713.864857ms INSERT - Takes(s): 0.7, Count: 1000, OPS: 1433.8, Avg(us): 1398, Min(us): 811, Max(us): 75278, 95th(us): 2000, 99th(us): 2000 table multi_capture_4.usertable not exists for 2-th check, retry later ***************** properties ***************** "mysql.user"="root" "requestdistribution"="uniform" "table"="a4" "updateproportion"="0" "threadcount"="2" "recordcount"="1000" "mysql.port"="4000" "workload"="core" "scanproportion"="0" "readproportion"="0" "insertproportion"="0" "dotransactions"="false" "mysql.host"="127.0.0.1" "readallfields"="true" "operationcount"="0" "mysql.db"="multi_rocks" ********************************************** table partition_table.finish_mark not exists for 3-th check, retry later Run finished, takes 543.149039ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1909.6, Avg(us): 1058, Min(us): 780, Max(us): 19430, 95th(us): 2000, 99th(us): 2000 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365842861521103719, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449575379289243654, "checkpoint_time": "2024-05-06 19:12:50.067", "error": null }' + echo '{ "upstream_id": 7365842861521103719, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449575379289243654, "checkpoint_time": "2024-05-06 19:12:50.067", "error": null }' { "upstream_id": 7365842861521103719, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449575379289243654, "checkpoint_time": "2024-05-06 19:12:50.067", "error": null } ++ echo '{' '"upstream_id":' 7365842861521103719, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449575379289243654, '"checkpoint_time":' '"2024-05-06' '19:12:50.067",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365842861521103719, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449575379289243654, '"checkpoint_time":' '"2024-05-06' '19:12:50.067",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12928.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name ***************** properties ***************** "readproportion"="0" "table"="a5" "mysql.host"="127.0.0.1" "dotransactions"="false" "updateproportion"="0" "readallfields"="true" "operationcount"="0" "recordcount"="1000" "scanproportion"="0" "workload"="core" "requestdistribution"="uniform" "mysql.port"="4000" "threadcount"="2" "insertproportion"="0" "mysql.user"="root" "mysql.db"="multi_rocks" ********************************************** table event_filter.t_truncate not exists for 4-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc0017f80f8} {Type:update Path:[Config SyncPointRetention] From: To:0xc0017f8138} {Type:update Path:[Config Consistent] From: To:0xc001b26af0} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Run finished, takes 547.04992ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1886.4, Avg(us): 1064, Min(us): 808, Max(us): 16870, 95th(us): 2000, 99th(us): 2000 table multi_capture_4.usertable not exists for 3-th check, retry later Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7365842861521103719,"namespace":"default","id":"custom-changefeed-name","sink_uri":"file:///tmp/tidb_cdc_test/cli_with_auth/storage_test/ticdc-cli-test-11714?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:12:34.867110854+08:00","start_ts":449575373679362050,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":0,"checkpoint_ts":449575379289243654,"checkpoint_time":"2024-05-06 19:12:50.067"} PASS coverage: 2.8% of statements in github.com/pingcap/tiflow/... table partition_table.finish_mark not exists for 4-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:56 GMT < Content-Length: 885 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dae99d0d-cac3-49e8-9015-869409659933 {"id":"dae99d0d-cac3-49e8-9015-869409659933","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993973} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a492ed2 dae99d0d-cac3-49e8-9015-869409659933 /tidb/cdc/default/default/upstream/7365842764385118364 {"id":7365842764385118364,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dae99d0d-cac3-49e8-9015-869409659933 {"id":"dae99d0d-cac3-49e8-9015-869409659933","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993973} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a492ed2 dae99d0d-cac3-49e8-9015-869409659933 /tidb/cdc/default/default/upstream/7365842764385118364 {"id":7365842764385118364,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dae99d0d-cac3-49e8-9015-869409659933 {"id":"dae99d0d-cac3-49e8-9015-869409659933","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993973} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a492ed2 dae99d0d-cac3-49e8-9015-869409659933 /tidb/cdc/default/default/upstream/7365842764385118364 {"id":7365842764385118364,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6062.out cli changefeed create --start-ts=449575367979040769 '--sink-uri=file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-3766?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error-2 Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7365842764385118364,"namespace":"default","id":"changefeed-error-2","sink_uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-3766?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:12:57.180325073+08:00","start_ts":449575367979040769,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575367979040769,"checkpoint_ts":449575367979040769,"checkpoint_time":"2024-05-06 19:12:06.922"} PASS Starting Upstream TiDB... coverage: 2.5% of statements in github.com/pingcap/tiflow/... table event_filter.t_truncate not exists for 5-th check, retry later Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12963.out cli changefeed --changefeed-id custom-changefeed-name resume table multi_rocks.finish_mark not exists for 1-th check, retry later table multi_capture_4.usertable not exists for 4-th check, retry later PASS table partition_table.finish_mark not exists for 5-th check, retry later coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s + info='{ "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449575367979040769, "checkpoint_time": "2024-05-06 19:12:06.922", "error": { "time": "2024-05-06T19:12:57.223554566+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575367979040769 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449575367979040769, "checkpoint_time": "2024-05-06 19:12:06.922", "error": { "time": "2024-05-06T19:12:57.223554566+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575367979040769 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449575367979040769, "checkpoint_time": "2024-05-06 19:12:06.922", "error": { "time": "2024-05-06T19:12:57.223554566+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575367979040769 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ echo '{' '"upstream_id":' 7365842764385118364, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449575367979040769, '"checkpoint_time":' '"2024-05-06' '19:12:06.922",' '"error":' '{' '"time":' '"2024-05-06T19:12:57.223554566+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575367979040769 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365842764385118364, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449575367979040769, '"checkpoint_time":' '"2024-05-06' '19:12:06.922",' '"error":' '{' '"time":' '"2024-05-06T19:12:57.223554566+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575367979040769 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575367979040769 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575367979040769 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6144.out cli changefeed remove -c changefeed-error-2 table event_filter.t_truncate exists table event_filter.t_alter exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_rocks.finish_mark not exists for 2-th check, retry later Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 449575367979040769 SinkURI: file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-3766?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table multi_capture_4.usertable not exists for 5-th check, retry later + set +x table event_filter.finish_mark exists check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 6-th check, retry later + set +x table multi_rocks.finish_mark not exists for 3-th check, retry later wait process cdc.test exit for 1-th time... table multi_capture_4.usertable exists check diff successfully ***************** properties ***************** "insertproportion"="0" "recordcount"="20" "scanproportion"="0" "mysql.db"="multi_capture_1" "mysql.host"="127.0.0.1" "operationcount"="0" "readallfields"="true" "requestdistribution"="uniform" "mysql.user"="root" "threadcount"="2" "workload"="core" "mysql.port"="4000" "dotransactions"="false" "readproportion"="0" "updateproportion"="0" ********************************************** Run finished, takes 12.036062ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1890.4, Avg(us): 1148, Min(us): 499, Max(us): 2403, 95th(us): 2000, 99th(us): 3000 ***************** properties ***************** "workload"="core" "recordcount"="20" "mysql.db"="multi_capture_2" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "operationcount"="0" "insertproportion"="0" "updateproportion"="0" "threadcount"="2" "scanproportion"="0" "readproportion"="0" "readallfields"="true" "mysql.user"="root" "dotransactions"="false" "mysql.port"="4000" ********************************************** Run finished, takes 16.106007ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1348.2, Avg(us): 1427, Min(us): 822, Max(us): 2547, 95th(us): 3000, 99th(us): 3000 ***************** properties ***************** "mysql.user"="root" "insertproportion"="0" "readproportion"="0" "workload"="core" "mysql.host"="127.0.0.1" "mysql.port"="4000" "operationcount"="0" "dotransactions"="false" "mysql.db"="multi_capture_3" "scanproportion"="0" "updateproportion"="0" "readallfields"="true" "recordcount"="20" "threadcount"="2" "requestdistribution"="uniform" ********************************************** Run finished, takes 10.914732ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2085.5, Avg(us): 1014, Min(us): 507, Max(us): 1883, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "requestdistribution"="uniform" "mysql.db"="multi_capture_4" "workload"="core" "readproportion"="0" "readallfields"="true" "mysql.host"="127.0.0.1" "updateproportion"="0" "mysql.port"="4000" "mysql.user"="root" "recordcount"="20" "insertproportion"="0" "threadcount"="2" "operationcount"="0" "dotransactions"="false" "scanproportion"="0" ********************************************** Run finished, takes 23.988058ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1315.0, Avg(us): 2330, Min(us): 585, Max(us): 8751, 95th(us): 9000, 99th(us): 9000 check diff failed 1-th time, retry later wait process cdc.test exit for 2-th time... check diff failed 2-th time, retry later + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365842861521103719, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575382435233798, "checkpoint_time": "2024-05-06 19:13:02.068", "error": null }' + echo '{ "upstream_id": 7365842861521103719, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575382435233798, "checkpoint_time": "2024-05-06 19:13:02.068", "error": null }' { "upstream_id": 7365842861521103719, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575382435233798, "checkpoint_time": "2024-05-06 19:13:02.068", "error": null } ++ echo '{' '"upstream_id":' 7365842861521103719, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575382435233798, '"checkpoint_time":' '"2024-05-06' '19:13:02.068",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365842861521103719, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575382435233798, '"checkpoint_time":' '"2024-05-06' '19:13:02.068",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13042.out cli changefeed --changefeed-id custom-changefeed-name remove VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366db0040014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35j, pid:4495, start at 2024-05-06 19:13:02.501879022 +0800 CST m=+5.385991421 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:02.509 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:02.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:02.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... table multi_rocks.finish_mark not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:13:03 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.61986200.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 449575382697377798 table partition_table.finish_mark not exists for 7-th check, retry later SinkURI: file:///tmp/tidb_cdc_test/cli_with_auth/storage_test/ticdc-cli-test-11714?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366db0040014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35j, pid:4495, start at 2024-05-06 19:13:02.501879022 +0800 CST m=+5.385991421 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:02.509 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:02.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:02.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366db17c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35j, pid:4570, start at 2024-05-06 19:13:02.585431572 +0800 CST m=+5.415924576 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:02.595 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:02.559 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:02.559 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x check diff failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366db1f80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5, pid:6584, start at 2024-05-06 19:13:02.636256118 +0800 CST m=+5.205752310 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:02.644 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:02.639 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:02.639 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366db1f80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5, pid:6584, start at 2024-05-06 19:13:02.636256118 +0800 CST m=+5.205752310 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:02.644 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:02.639 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:02.639 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366db4340013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5, pid:6659, start at 2024-05-06 19:13:02.752405161 +0800 CST m=+5.272618756 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:02.758 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:02.733 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:02.733 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table partition_table.finish_mark not exists for 8-th check, retry later table multi_rocks.finish_mark not exists for 5-th check, retry later check diff failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:13:07 GMT < Content-Length: 885 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/63d3acec-9650-4f17-a1cc-9552c84b47a5 {"id":"63d3acec-9650-4f17-a1cc-9552c84b47a5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993984} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a492f4a 63d3acec-9650-4f17-a1cc-9552c84b47a5 /tidb/cdc/default/default/upstream/7365842764385118364 {"id":7365842764385118364,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/63d3acec-9650-4f17-a1cc-9552c84b47a5 {"id":"63d3acec-9650-4f17-a1cc-9552c84b47a5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993984} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a492f4a 63d3acec-9650-4f17-a1cc-9552c84b47a5 /tidb/cdc/default/default/upstream/7365842764385118364 {"id":7365842764385118364,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/63d3acec-9650-4f17-a1cc-9552c84b47a5 {"id":"63d3acec-9650-4f17-a1cc-9552c84b47a5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993984} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a492f4a 63d3acec-9650-4f17-a1cc-9552c84b47a5 /tidb/cdc/default/default/upstream/7365842764385118364 {"id":7365842764385118364,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6248.out cli changefeed create --start-ts=0 '--sink-uri=file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-3766?protocol=canal-json&enable-tidb-extension=true' -c changefeed-initialize-error check diff failed 4-th time, retry later Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7365842764385118364,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-3766?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:13:07.532252509+08:00","start_ts":449575383838752770,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575383838752770,"checkpoint_ts":449575383838752770,"checkpoint_time":"2024-05-06 19:13:07.422"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [Mon May 6 19:13:07 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.79897991.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 19:13:07 CST 2024] <<<<<< START cdc server in processor_etcd_worker_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/orchestrator/ProcessorEtcdDelay=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_etcd_worker_delay.58865888.out server --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.finish_mark not exists for 9-th check, retry later table multi_rocks.finish_mark not exists for 6-th check, retry later check diff failed 4-th time, retry later changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13114.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/cli_with_auth/storage_test/ticdc-cli-test-11714?protocol=canal-json&enable-tidb-extension=true' --tz=Asia/Shanghai -c=custom-changefeed-name + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365842861521103719,"namespace":"default","id":"custom-changefeed-name","sink_uri":"file:///tmp/tidb_cdc_test/cli_with_auth/storage_test/ticdc-cli-test-11714?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:13:09.215253224+08:00","start_ts":449575384269979654,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575384269979654,"checkpoint_ts":449575384269979654,"checkpoint_time":"2024-05-06 19:13:09.067"} PASS + info='{ "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449575383838752770, "checkpoint_time": "2024-05-06 19:13:07.422", "error": { "time": "2024-05-06T19:13:07.722110427+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449575383838752770, "checkpoint_time": "2024-05-06 19:13:07.422", "error": { "time": "2024-05-06T19:13:07.722110427+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449575383838752770, "checkpoint_time": "2024-05-06 19:13:07.422", "error": { "time": "2024-05-06T19:13:07.722110427+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365842764385118364, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449575383838752770, '"checkpoint_time":' '"2024-05-06' '19:13:07.422",' '"error":' '{' '"time":' '"2024-05-06T19:13:07.722110427+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 5-th time, retry later table partition_table.finish_mark not exists for 10-th check, retry later table multi_rocks.finish_mark not exists for 7-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:13:10 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b873c49c-9d56-4cbb-843c-54228f326b5b {"id":"b873c49c-9d56-4cbb-843c-54228f326b5b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993987} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b3f42cb b873c49c-9d56-4cbb-843c-54228f326b5b /tidb/cdc/default/default/upstream/7365843031638247133 {"id":7365843031638247133,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b873c49c-9d56-4cbb-843c-54228f326b5b {"id":"b873c49c-9d56-4cbb-843c-54228f326b5b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993987} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b3f42cb b873c49c-9d56-4cbb-843c-54228f326b5b /tidb/cdc/default/default/upstream/7365843031638247133 {"id":7365843031638247133,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b873c49c-9d56-4cbb-843c-54228f326b5b {"id":"b873c49c-9d56-4cbb-843c-54228f326b5b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993987} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b3f42cb b873c49c-9d56-4cbb-843c-54228f326b5b /tidb/cdc/default/default/upstream/7365843031638247133 {"id":7365843031638247133,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/processor_etcd_worker_delay + sink_uri='file:///tmp/tidb_cdc_test/processor_etcd_worker_delay/storage_test/processor-etcd-delay-4261?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/processor_etcd_worker_delay ++ date + echo '[Mon May 6 19:13:10 CST 2024] <<<<<< START storage consumer in processor_etcd_worker_delay case >>>>>>' [Mon May 6 19:13:10 CST 2024] <<<<<< START storage consumer in processor_etcd_worker_delay case >>>>>> + cd /tmp/tidb_cdc_test/processor_etcd_worker_delay + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/processor_etcd_worker_delay + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/processor_etcd_worker_delay/storage_test/processor-etcd-delay-4261?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' check diff successfully + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:13:10 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40ca7313-d261-496e-a2c9-e5df37ce6a78 {"id":"40ca7313-d261-496e-a2c9-e5df37ce6a78","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993988} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b446ec8 40ca7313-d261-496e-a2c9-e5df37ce6a78 /tidb/cdc/default/default/upstream/7365843035333155358 {"id":7365843035333155358,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40ca7313-d261-496e-a2c9-e5df37ce6a78 {"id":"40ca7313-d261-496e-a2c9-e5df37ce6a78","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993988} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b446ec8 40ca7313-d261-496e-a2c9-e5df37ce6a78 /tidb/cdc/default/default/upstream/7365843035333155358 {"id":7365843035333155358,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40ca7313-d261-496e-a2c9-e5df37ce6a78 {"id":"40ca7313-d261-496e-a2c9-e5df37ce6a78","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714993988} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b446ec8 40ca7313-d261-496e-a2c9-e5df37ce6a78 /tidb/cdc/default/default/upstream/7365843035333155358 {"id":7365843035333155358,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/processor_stop_delay + sink_uri='file:///tmp/tidb_cdc_test/processor_stop_delay/storage_test/ticdc-processor-stop-delay-11452?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/processor_stop_delay ++ date + echo '[Mon May 6 19:13:11 CST 2024] <<<<<< START storage consumer in processor_stop_delay case >>>>>>' [Mon May 6 19:13:11 CST 2024] <<<<<< START storage consumer in processor_stop_delay case >>>>>> + cd /tmp/tidb_cdc_test/processor_stop_delay + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/processor_stop_delay + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/processor_stop_delay/storage_test/ticdc-processor-stop-delay-11452?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' [2024/05/06 19:13:01.429 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/06 19:13:01.429 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7365842307661950878] [2024/05/06 19:13:01.429 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 19:13:01.429 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/06 19:13:01.430 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/06 19:13:01.430 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 19:13:01.433 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/06 19:13:01.433 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7365842307661950878] [2024/05/06 19:13:01.433 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 19:13:01.433 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/06 19:13:01.434 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/06 19:13:01.434 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 19:13:01.434 +08:00] [INFO] [tikv_driver.go:197] ["using API V1."] [2024/05/06 19:13:01.435 +08:00] [INFO] [main.go:180] ["genLock started"] [2024/05/06 19:13:01.436 +08:00] [INFO] [store_cache.go:477] ["change store resolve state"] [store=1] [addr=127.0.0.1:20160] [from=unresolved] [to=resolved] [liveness-state=reachable] [2024/05/06 19:13:01.508 +08:00] [INFO] [region_request.go:1688] ["send request meet region error without retry"] [req-ts=449575382260908045] [req-type=Prewrite] [region="{ region id: 26, ver: 58, confVer: 1 }"] [replica-read-type=leader] [stale-read=false] [request-sender="{rpcError:, replicaSelector: replicaSelectorV2{replicaReadType: leader, attempts: 6, cacheRegionIsValid: false, replicaStatus: [peer: 27, store: 1, isEpochStale: false, attempts: 6, replica-epoch: 0, store-epoch: 0, store-state: resolved, store-liveness-state: reachable]}}"] [retry-times=5] [total-backoff-ms=62] [total-backoff-times=5] [max-exec-timeout-ms=20000] [total-region-errors="27-read_index_not_ready:5, 27-epoch_not_match:1"] wait process cdc.test exit for 1-th time... [2024/05/06 19:13:11.454 +08:00] [INFO] [main.go:196] ["genLock done"] [2024/05/06 19:13:11.454 +08:00] [INFO] [pd_service_discovery.go:550] ["[pd] exit member loop due to context canceled"] [2024/05/06 19:13:11.454 +08:00] [INFO] [tso_dispatcher.go:268] ["exit tso dispatcher loop"] [2024/05/06 19:13:11.454 +08:00] [INFO] [tso_dispatcher.go:455] ["[tso] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2024/05/06 19:13:11.454 +08:00] [INFO] [tso_dispatcher.go:214] ["exit tso requests cancel loop"] [2024/05/06 19:13:11.454 +08:00] [INFO] [tso_dispatcher.go:380] ["[tso] exit tso dispatcher"] [dc-location=global] [2024/05/06 19:13:11.454 +08:00] [INFO] [resource_manager_client.go:295] ["[resource manager] exit resource token dispatcher"] [2024/05/06 19:13:11.454 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/06 19:13:11.454 +08:00] [INFO] [tso_client.go:140] ["closing tso client"] [2024/05/06 19:13:11.454 +08:00] [INFO] [tso_client.go:145] ["close tso client"] [2024/05/06 19:13:11.454 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/06 19:13:11.454 +08:00] [INFO] [tso_client.go:155] ["tso client is closed"] [2024/05/06 19:13:11.454 +08:00] [INFO] [pd_service_discovery.go:637] ["[pd] close pd service discovery client"] check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449575383838752770, "checkpoint_time": "2024-05-06 19:13:07.422", "error": { "time": "2024-05-06T19:13:07.722110427+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449575383838752770, "checkpoint_time": "2024-05-06 19:13:07.422", "error": { "time": "2024-05-06T19:13:07.722110427+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449575383838752770, "checkpoint_time": "2024-05-06 19:13:07.422", "error": { "time": "2024-05-06T19:13:07.722110427+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365842764385118364, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449575383838752770, '"checkpoint_time":' '"2024-05-06' '19:13:07.422",' '"error":' '{' '"time":' '"2024-05-06T19:13:07.722110427+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later wait process cdc.test exit for 2-th time... table partition_table.finish_mark not exists for 11-th check, retry later check diff failed 6-th time, retry later table multi_rocks.finish_mark not exists for 8-th check, retry later table processor_stop_delay.t not exists for 1-th check, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:13:12 CST 2024] <<<<<< run test case multi_capture success! >>>>>> check diff failed 7-th time, retry later table multi_rocks.finish_mark not exists for 9-th check, retry later + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365842861521103719, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575385331662850, "checkpoint_time": "2024-05-06 19:13:13.117", "error": null }' + echo '{ "upstream_id": 7365842861521103719, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575385331662850, "checkpoint_time": "2024-05-06 19:13:13.117", "error": null }' { "upstream_id": 7365842861521103719, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575385331662850, "checkpoint_time": "2024-05-06 19:13:13.117", "error": null } ++ echo '{' '"upstream_id":' 7365842861521103719, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575385331662850, '"checkpoint_time":' '"2024-05-06' '19:13:13.117",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365842861521103719, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575385331662850, '"checkpoint_time":' '"2024-05-06' '19:13:13.117",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] table processor_stop_delay.t not exists for 2-th check, retry later table partition_table.finish_mark not exists for 12-th check, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13235.out cli unsafe delete-service-gc-safepoint Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff failed 8-th time, retry later check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449575385896845316, "checkpoint_time": "2024-05-06 19:13:15.273", "error": null }' + echo '{ "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449575385896845316, "checkpoint_time": "2024-05-06 19:13:15.273", "error": null }' { "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449575385896845316, "checkpoint_time": "2024-05-06 19:13:15.273", "error": null } ++ echo '{' '"upstream_id":' 7365842764385118364, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449575385896845316, '"checkpoint_time":' '"2024-05-06' '19:13:15.273",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365842764385118364, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449575385896845316, '"checkpoint_time":' '"2024-05-06' '19:13:15.273",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6421.out cli changefeed pause -c changefeed-initialize-error table processor_stop_delay.t not exists for 3-th check, retry later table partition_table.finish_mark not exists for 13-th check, retry later table multi_rocks.finish_mark not exists for 10-th check, retry later PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13276.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 reset and all metadata truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449575385896845316, "checkpoint_time": "2024-05-06 19:13:15.273", "error": { "time": "2024-05-06T19:13:07.722110427+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449575385896845316, "checkpoint_time": "2024-05-06 19:13:15.273", "error": { "time": "2024-05-06T19:13:07.722110427+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449575385896845316, "checkpoint_time": "2024-05-06 19:13:15.273", "error": { "time": "2024-05-06T19:13:07.722110427+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365842764385118364, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449575385896845316, '"checkpoint_time":' '"2024-05-06' '19:13:15.273",' '"error":' '{' '"time":' '"2024-05-06T19:13:07.722110427+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365842764385118364, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449575385896845316, '"checkpoint_time":' '"2024-05-06' '19:13:15.273",' '"error":' '{' '"time":' '"2024-05-06T19:13:07.722110427+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6496.out cli changefeed resume -c changefeed-initialize-error table test.t2 not exists for 1-th check, retry later table partition_table.finish_mark not exists for 14-th check, retry later table multi_rocks.finish_mark not exists for 11-th check, retry later check diff failed 9-th time, retry later table processor_stop_delay.t not exists for 4-th check, retry later PASS table processor_delay.t1 not exists for 1-th check, retry later coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x table test.t2 not exists for 2-th check, retry later table processor_delay.t1 not exists for 2-th check, retry later table partition_table.finish_mark exists check diff successfully table multi_rocks.finish_mark not exists for 12-th check, retry later check diff successfully + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s wait process cdc.test exit for 1-th time... table processor_stop_delay.t not exists for 5-th check, retry later + info='{ "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449575385896845316, "checkpoint_time": "2024-05-06 19:13:15.273", "error": null }' + echo '{ "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449575385896845316, "checkpoint_time": "2024-05-06 19:13:15.273", "error": null }' { "upstream_id": 7365842764385118364, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449575385896845316, "checkpoint_time": "2024-05-06 19:13:15.273", "error": null } ++ echo '{' '"upstream_id":' 7365842764385118364, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449575385896845316, '"checkpoint_time":' '"2024-05-06' '19:13:15.273",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365842764385118364, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449575385896845316, '"checkpoint_time":' '"2024-05-06' '19:13:15.273",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6563.out cli changefeed remove -c changefeed-initialize-error wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 449575385896845316 SinkURI: file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-3766?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:13:21 CST 2024] <<<<<< run test case event_filter success! >>>>>> table test.t2 not exists for 3-th check, retry later table processor_delay.t1 exists table processor_delay.t2 not exists for 1-th check, retry later wait process cdc.test exit for 3-th time... table multi_rocks.finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13393.out cli unsafe resolve-lock --region=72 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:13:22 CST 2024] <<<<<< run test case csv_storage_partition_table success! >>>>>> table processor_stop_delay.t exists check diff successfully wait process cdc.test exit for 1-th time... + set +x wait process cdc.test exit for 2-th time... wait process cdc.test exit for 1-th time... table test.t2 not exists for 4-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13425.out cli unsafe resolve-lock --region=72 --ts=449575386786562051 wait process cdc.test exit for 3-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:13:24 CST 2024] <<<<<< run test case multi_rocks success! >>>>>> check_changefeed_state http://127.0.0.1:2379 c25efb9a-f1b6-4386-9f87-0bb59d045bfb finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=c25efb9a-f1b6-4386-9f87-0bb59d045bfb + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c c25efb9a-f1b6-4386-9f87-0bb59d045bfb -s + info='{ "upstream_id": 7365842612003849854, "namespace": "default", "id": "c25efb9a-f1b6-4386-9f87-0bb59d045bfb", "state": "finished", "checkpoint_tso": 449575383244472326, "checkpoint_time": "2024-05-06 19:13:05.155", "error": null }' + echo '{ "upstream_id": 7365842612003849854, "namespace": "default", "id": "c25efb9a-f1b6-4386-9f87-0bb59d045bfb", "state": "finished", "checkpoint_tso": 449575383244472326, "checkpoint_time": "2024-05-06 19:13:05.155", "error": null }' { "upstream_id": 7365842612003849854, "namespace": "default", "id": "c25efb9a-f1b6-4386-9f87-0bb59d045bfb", "state": "finished", "checkpoint_tso": 449575383244472326, "checkpoint_time": "2024-05-06 19:13:05.155", "error": null } ++ echo '{' '"upstream_id":' 7365842612003849854, '"namespace":' '"default",' '"id":' '"c25efb9a-f1b6-4386-9f87-0bb59d045bfb",' '"state":' '"finished",' '"checkpoint_tso":' 449575383244472326, '"checkpoint_time":' '"2024-05-06' '19:13:05.155",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7365842612003849854, '"namespace":' '"default",' '"id":' '"c25efb9a-f1b6-4386-9f87-0bb59d045bfb",' '"state":' '"finished",' '"checkpoint_tso":' 449575383244472326, '"checkpoint_time":' '"2024-05-06' '19:13:05.155",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:13:19 CST 2024] <<<<<< run test case changefeed_finish success! >>>>>> table processor_delay.t2 not exists for 2-th check, retry later PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:13:24 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> table test.t2 not exists for 5-th check, retry later + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 149k 4332 --:--:-- --:--:-- --:--:-- 207k { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" table processor_delay.t2 not exists for 3-th check, retry later check diff failed 2-th time, retry later table test.t2 not exists for 6-th check, retry later table processor_delay.t2 not exists for 4-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh using Sink-Type: storage... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status_with_redo + CDC_BINARY=cdc.test + SINK_TYPE=storage + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo The 1 times to try to start tidb cluster... check diff failed 3-th time, retry later } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 183k 0 --:--:-- --:--:-- --:--:-- 177k { "version": "v8.2.0-alpha-25-g3c107706d", "git_hash": "3c107706d2612c5f422cde6480fd92425795842e", "id": "3692bedb-8088-46c8-bc5a-d02399e36dd9", "pid": 12417, "is_owner": true }wait process cdc.test exit for 1-th time... table test.t2 not exists for 7-th check, retry later wait process cdc.test exit for 2-th time... table processor_delay.t2 not exists for 5-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:13:29 CST 2024] <<<<<< run test case cli_with_auth success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... check diff failed 4-th time, retry later table test.t2 not exists for 8-th check, retry later table processor_delay.t2 exists table processor_delay.t3 exists table processor_delay.t4 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... check diff failed 5-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... table test.t2 not exists for 9-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/sql_mode/run.sh using Sink-Type: storage... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/sql_mode/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/sql_mode ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/sql_mode + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/sql_mode/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/sql_mode + CDC_BINARY=cdc.test + SINK_TYPE=storage + CDC_COUNT=3 + DB_COUNT=4 + rm -rf /tmp/tidb_cdc_test/sql_mode + mkdir -p /tmp/tidb_cdc_test/sql_mode + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t4 not exists for 2-th check, retry later check diff failed 6-th time, retry later table test.t2 not exists for 10-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t4 not exists for 3-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc already exists) start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } check diff failed 7-th time, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] // withEnv Verifying downstream PD is started... [Pipeline] } [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc already exists) [Pipeline] // stage [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] // dir [Pipeline] } table test.t2 exists check diff successfully [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } wait process cdc.test exit for 1-th time... table processor_delay.t4 not exists for 4-th check, retry later [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // stage [Pipeline] } check diff failed 8-th time, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:13:39 CST 2024] <<<<<< run test case resolve_lock success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_delay.t4 not exists for 5-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366feb280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:10307, start at 2024-05-06 19:13:39.054524681 +0800 CST m=+5.446264330 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:39.061 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:39.018 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:39.018 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... check diff failed 9-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t4 exists table processor_delay.t5 exists table processor_delay.t6 not exists for 1-th check, retry later Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366feb280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:10307, start at 2024-05-06 19:13:39.054524681 +0800 CST m=+5.446264330 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:39.061 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:39.018 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:39.018 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366fecac0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:10391, start at 2024-05-06 19:13:39.135267206 +0800 CST m=+5.469184554 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:39.142 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:39.115 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:39.115 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11708.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 2-th time... table processor_delay.t6 not exists for 2-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:13:44 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + set +x + tso='449575393497710593 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575393497710593 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575393497710593 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Mon May 6 19:13:45 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1174011742.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_delay.t6 not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t6 not exists for 4-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:13:48 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d7591fb4-8eef-4118-baf2-a42b4462ad40 {"id":"d7591fb4-8eef-4118-baf2-a42b4462ad40","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994026} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9bd79ed6 d7591fb4-8eef-4118-baf2-a42b4462ad40 /tidb/cdc/default/default/upstream/7365843188522606704 {"id":7365843188522606704,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d7591fb4-8eef-4118-baf2-a42b4462ad40 {"id":"d7591fb4-8eef-4118-baf2-a42b4462ad40","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994026} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9bd79ed6 d7591fb4-8eef-4118-baf2-a42b4462ad40 /tidb/cdc/default/default/upstream/7365843188522606704 {"id":7365843188522606704,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d7591fb4-8eef-4118-baf2-a42b4462ad40 {"id":"d7591fb4-8eef-4118-baf2-a42b4462ad40","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994026} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9bd79ed6 d7591fb4-8eef-4118-baf2-a42b4462ad40 /tidb/cdc/default/default/upstream/7365843188522606704 {"id":7365843188522606704,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575393497710593 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11794.out cli changefeed create --start-ts=449575393497710593 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365843188522606704,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:13:49.281096524+08:00","start_ts":449575393497710593,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575393497710593,"checkpoint_ts":449575393497710593,"checkpoint_time":"2024-05-06 19:13:44.268"} PASS ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.5% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36707050000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-722-gnh4g, pid:4052, start at 2024-05-06 19:13:47.555522231 +0800 CST m=+6.240206034 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:47.567 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:47.540 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:47.540 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36707050000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-722-gnh4g, pid:4052, start at 2024-05-06 19:13:47.555522231 +0800 CST m=+6.240206034 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:47.567 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:47.540 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:47.540 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3670746c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-722-gnh4g, pid:4103, start at 2024-05-06 19:13:47.836954447 +0800 CST m=+6.451383993 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:47.847 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:47.853 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:47.853 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table processor_delay.t6 not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 1564 0 --:--:-- --:--:-- --:--:-- 1567 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 19:13:44.268","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-06 19:13:50.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:13:44.268","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:13:50.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:13:44.268","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:13:50.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-06 19:13:44.268' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:13:44.268","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:13:50.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:13:44.268","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:13:50.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-06 19:13:51' + echo 'sink_checkpoint_ts is 2024-05-06' 19:13:44.268 sink_checkpoint_ts is 2024-05-06 19:13:44.268 ++ date -d '2024-05-06 19:13:44.268' +%s + checkpoint_timestamp=1714994024 ++ date -d '2024-05-06 19:13:51' +%s + current_timestamp=1714994031 + '[' 7 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36709760001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:9583, start at 2024-05-06 19:13:50.085722356 +0800 CST m=+5.197441696 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:50.092 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:50.090 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:50.090 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36709760001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:9583, start at 2024-05-06 19:13:50.085722356 +0800 CST m=+5.197441696 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:50.092 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:50.090 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:50.090 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367098140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:9664, start at 2024-05-06 19:13:50.11866794 +0800 CST m=+5.175877516 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:50.129 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:50.085 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:50.085 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sql_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sql_mode/tiflash/log/error.log =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 19:13:51 CST 2024] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.53745376.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_delay.t6 exists table processor_delay.t7 exists table processor_delay.t8 not exists for 1-th check, retry later arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + trap stop_tidb_cluster EXIT + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 3306 + cd /tmp/tidb_cdc_test/sql_mode ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.10925.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t1 exists + sleep 5 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3670ca5c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms, pid:7529, start at 2024-05-06 19:13:53.32464156 +0800 CST m=+5.630012733 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:53.334 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:53.303 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:53.303 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t8 not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449575395941679105 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575395941679105 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575395941679105 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Mon May 6 19:13:55 CST 2024] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.1096510967.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:13:54 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a8f79a61-0b5b-4a7f-9694-9d1bda2cb6a3 {"id":"a8f79a61-0b5b-4a7f-9694-9d1bda2cb6a3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994032} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9bed91d3 a8f79a61-0b5b-4a7f-9694-9d1bda2cb6a3 /tidb/cdc/default/default/upstream/7365843213354544958 {"id":7365843213354544958,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a8f79a61-0b5b-4a7f-9694-9d1bda2cb6a3 {"id":"a8f79a61-0b5b-4a7f-9694-9d1bda2cb6a3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994032} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9bed91d3 a8f79a61-0b5b-4a7f-9694-9d1bda2cb6a3 /tidb/cdc/default/default/upstream/7365843213354544958 {"id":7365843213354544958,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a8f79a61-0b5b-4a7f-9694-9d1bda2cb6a3 {"id":"a8f79a61-0b5b-4a7f-9694-9d1bda2cb6a3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994032} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9bed91d3 a8f79a61-0b5b-4a7f-9694-9d1bda2cb6a3 /tidb/cdc/default/default/upstream/7365843213354544958 {"id":7365843213354544958,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 11e60ca3-a76b-46cb-9a41-d52a1adea0cc Info: {"upstream_id":7365843213354544958,"namespace":"default","id":"11e60ca3-a76b-46cb-9a41-d52a1adea0cc","sink_uri":"file:///tmp/tidb_cdc_test/force_replicate_table/storage_test/ticdc-force_replicate_table-test-28626?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:13:55.097153087+08:00","start_ts":449575395469557761,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575395469557761,"checkpoint_ts":449575395469557761,"checkpoint_time":"2024-05-06 19:13:51.790"} + workdir=/tmp/tidb_cdc_test/force_replicate_table + sink_uri='file:///tmp/tidb_cdc_test/force_replicate_table/storage_test/ticdc-force_replicate_table-test-28626?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/force_replicate_table/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/force_replicate_table ++ date + echo '[Mon May 6 19:13:55 CST 2024] <<<<<< START storage consumer in force_replicate_table case >>>>>>' [Mon May 6 19:13:55 CST 2024] <<<<<< START storage consumer in force_replicate_table case >>>>>> + cd /tmp/tidb_cdc_test/force_replicate_table + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/force_replicate_table/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/force_replicate_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/force_replicate_table/storage_test/ticdc-force_replicate_table-test-28626?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/force_replicate_table/conf/changefeed.toml VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3670ca5c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms, pid:7529, start at 2024-05-06 19:13:53.32464156 +0800 CST m=+5.630012733 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:53.334 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:53.303 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:53.303 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3670cbc8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms, pid:7610, start at 2024-05-06 19:13:53.410137578 +0800 CST m=+5.663303015 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:53.417 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:53.394 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:53.394 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table processor_delay.t8 not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3670f6ac0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:14386, start at 2024-05-06 19:13:56.165199632 +0800 CST m=+5.536774212 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:56.174 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:56.139 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:56.139 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table force_replicate_table.t0 not exists for 1-th check, retry later table processor_delay.t8 not exists for 4-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:13:58 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6 {"id":"6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994035} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9bffbed3 6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6 /tidb/cdc/default/default/upstream/7365843235085893686 {"id":7365843235085893686,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6 {"id":"6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994035} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9bffbed3 6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6 /tidb/cdc/default/default/upstream/7365843235085893686 {"id":7365843235085893686,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6 {"id":"6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994035} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9bffbed3 6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6 /tidb/cdc/default/default/upstream/7365843235085893686 {"id":7365843235085893686,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575395941679105 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.11019.out cli changefeed create --start-ts=449575395941679105 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.8887.out cli tso query --pd=http://127.0.0.1:2379 Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365843235085893686,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:13:58.590151283+08:00","start_ts":449575395941679105,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575395941679105,"checkpoint_ts":449575395941679105,"checkpoint_time":"2024-05-06 19:13:53.591"} PASS Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3670f6ac0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:14386, start at 2024-05-06 19:13:56.165199632 +0800 CST m=+5.536774212 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:56.174 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:56.139 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:56.139 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3670f6a00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:14471, start at 2024-05-06 19:13:56.158613304 +0800 CST m=+5.477759432 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:56.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:56.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:56.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } coverage: 2.4% of statements in github.com/pingcap/tiflow/... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2577 0 --:--:-- --:--:-- --:--:-- 2585 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:13:56.668","puller_resolved_ts":"2024-05-06 19:13:51.018","last_synced_ts":"2024-05-06 19:13:51.118","now_ts":"2024-05-06 19:13:58.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:13:56.668","puller_resolved_ts":"2024-05-06' '19:13:51.018","last_synced_ts":"2024-05-06' '19:13:51.118","now_ts":"2024-05-06' '19:13:58.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:13:56.668","puller_resolved_ts":"2024-05-06' '19:13:51.018","last_synced_ts":"2024-05-06' '19:13:51.118","now_ts":"2024-05-06' '19:13:58.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 table force_replicate_table.t0 not exists for 2-th check, retry later table processor_delay.t8 not exists for 5-th check, retry later Starting Upstream TiDB... + set +x + run_sql 'use test; create table t1(id bigint primary key, a text, b text as ((regexp_replace(a, '\''^[1-9]\d{9,29}$'\'', '\''aaaaa'\''))), c text); insert into t1 (id, a, c) values(1,123456, '\''ab\\\\c'\''); insert into t1 (id, a, c) values(2,1234567890123, '\''ab\\c'\'');' 127.0.0.1 4000 + set +x + tso='449575397268652033 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575397268652033 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:14:00 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.89638965.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + '[' storage == mysql ']' + stop_tidb_cluster table force_replicate_table.t0 not exists for 3-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.15786.out cli tso query --pd=http://127.0.0.1:2379 table processor_delay.t8 exists table processor_delay.t9 exists table processor_delay.t10 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... + set +x + tso='449575397959401473 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575397959401473 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:14:02 CST 2024] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.1581915821.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:14:03 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5b2f800f-c360-4b15-8877-5c02d8027bfd {"id":"5b2f800f-c360-4b15-8877-5c02d8027bfd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994040} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c04dad5 5b2f800f-c360-4b15-8877-5c02d8027bfd /tidb/cdc/default/default/upstream/7365843247057059262 {"id":7365843247057059262,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5b2f800f-c360-4b15-8877-5c02d8027bfd {"id":"5b2f800f-c360-4b15-8877-5c02d8027bfd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994040} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c04dad5 5b2f800f-c360-4b15-8877-5c02d8027bfd /tidb/cdc/default/default/upstream/7365843247057059262 {"id":7365843247057059262,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5b2f800f-c360-4b15-8877-5c02d8027bfd {"id":"5b2f800f-c360-4b15-8877-5c02d8027bfd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994040} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c04dad5 5b2f800f-c360-4b15-8877-5c02d8027bfd /tidb/cdc/default/default/upstream/7365843247057059262 {"id":7365843247057059262,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9019.out cli changefeed create --start-ts=449575397268652033 '--sink-uri=file:///tmp/tidb_cdc_test/ddl_sequence/storage_test/ticdc-ddl-sequence-test-5155?protocol=canal-json&enable-tidb-extension=true' table force_replicate_table.t0 not exists for 4-th check, retry later Create changefeed successfully! ID: b2e0e9ff-77e4-4cb0-873a-5424db20dab3 Info: {"upstream_id":7365843247057059262,"namespace":"default","id":"b2e0e9ff-77e4-4cb0-873a-5424db20dab3","sink_uri":"file:///tmp/tidb_cdc_test/ddl_sequence/storage_test/ticdc-ddl-sequence-test-5155?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:14:03.694066742+08:00","start_ts":449575397268652033,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575397268652033,"checkpoint_ts":449575397268652033,"checkpoint_time":"2024-05-06 19:13:58.653"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table processor_delay.t10 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... + set +x + workdir=/tmp/tidb_cdc_test/ddl_sequence + sink_uri='file:///tmp/tidb_cdc_test/ddl_sequence/storage_test/ticdc-ddl-sequence-test-5155?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/ddl_sequence ++ date + echo '[Mon May 6 19:14:05 CST 2024] <<<<<< START storage consumer in ddl_sequence case >>>>>>' [Mon May 6 19:14:05 CST 2024] <<<<<< START storage consumer in ddl_sequence case >>>>>> + cd /tmp/tidb_cdc_test/ddl_sequence + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/ddl_sequence + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/ddl_sequence/storage_test/ticdc-ddl-sequence-test-5155?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table force_replicate_table.t0 not exists for 5-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:14:05 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/acbb438e-e092-44a4-ab6f-3359b04a73a6 {"id":"acbb438e-e092-44a4-ab6f-3359b04a73a6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994043} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c124ad1 acbb438e-e092-44a4-ab6f-3359b04a73a6 /tidb/cdc/default/default/upstream/7365843262892684683 {"id":7365843262892684683,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/acbb438e-e092-44a4-ab6f-3359b04a73a6 {"id":"acbb438e-e092-44a4-ab6f-3359b04a73a6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994043} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c124ad1 acbb438e-e092-44a4-ab6f-3359b04a73a6 /tidb/cdc/default/default/upstream/7365843262892684683 {"id":7365843262892684683,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/acbb438e-e092-44a4-ab6f-3359b04a73a6 {"id":"acbb438e-e092-44a4-ab6f-3359b04a73a6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994043} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c124ad1 acbb438e-e092-44a4-ab6f-3359b04a73a6 /tidb/cdc/default/default/upstream/7365843262892684683 {"id":7365843262892684683,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.15881.out cli changefeed create --start-ts=449575397959401473 '--sink-uri=file:///tmp/tidb_cdc_test/savepoint/storage_test/ticdc-savepoint-test-7657?protocol=canal-json&enable-tidb-extension=true' table processor_delay.t10 not exists for 3-th check, retry later Create changefeed successfully! ID: c25d3a9d-5813-4321-9fde-f47c0e532398 Info: {"upstream_id":7365843262892684683,"namespace":"default","id":"c25d3a9d-5813-4321-9fde-f47c0e532398","sink_uri":"file:///tmp/tidb_cdc_test/savepoint/storage_test/ticdc-savepoint-test-7657?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:14:06.272664241+08:00","start_ts":449575397959401473,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575397959401473,"checkpoint_ts":449575397959401473,"checkpoint_time":"2024-05-06 19:14:01.288"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table force_replicate_table.t0 exists table force_replicate_table.t1 exists table force_replicate_table.t2 exists table force_replicate_table.t3 exists table force_replicate_table.t4 exists table force_replicate_table.t5 not exists for 1-th check, retry later table ddl_sequence.finish_mark not exists for 1-th check, retry later + set +x + workdir=/tmp/tidb_cdc_test/savepoint + sink_uri='file:///tmp/tidb_cdc_test/savepoint/storage_test/ticdc-savepoint-test-7657?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/savepoint ++ date + echo '[Mon May 6 19:14:07 CST 2024] <<<<<< START storage consumer in savepoint case >>>>>>' [Mon May 6 19:14:07 CST 2024] <<<<<< START storage consumer in savepoint case >>>>>> + cd /tmp/tidb_cdc_test/savepoint + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/savepoint + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/savepoint/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/savepoint/storage_test/ticdc-savepoint-test-7657?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table processor_delay.t10 not exists for 4-th check, retry later table savepoint.finish_mark not exists for 1-th check, retry later Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367196ec0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c8j-cs84c, pid:22438, start at 2024-05-06 19:14:06.4346658 +0800 CST m=+6.135226017 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:06.445 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:06.444 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:06.444 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367196ec0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c8j-cs84c, pid:22438, start at 2024-05-06 19:14:06.4346658 +0800 CST m=+6.135226017 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:06.445 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:06.444 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:06.444 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367198740008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c8j-cs84c, pid:22516, start at 2024-05-06 19:14:06.503962077 +0800 CST m=+6.130346151 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:06.512 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:06.493 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:06.493 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/move_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/move_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/move_table/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table force_replicate_table.t5 not exists for 2-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_sequence.finish_mark not exists for 2-th check, retry later table processor_delay.t10 not exists for 5-th check, retry later table savepoint.finish_mark not exists for 2-th check, retry later table force_replicate_table.t5 not exists for 3-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.23782.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_sequence.finish_mark not exists for 3-th check, retry later table processor_delay.t10 exists table processor_delay.t11 exists table processor_delay.t12 not exists for 1-th check, retry later table savepoint.finish_mark not exists for 3-th check, retry later + set +x + tso='449575400516878337 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575400516878337 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... table ddl_sequence.finish_mark not exists for 4-th check, retry later table force_replicate_table.t5 not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "insertproportion"="0" "readallfields"="true" "recordcount"="10000" "workload"="core" "scanproportion"="0" "requestdistribution"="uniform" "mysql.db"="move_table" "updateproportion"="0" "mysql.user"="root" "readproportion"="0" "mysql.host"="127.0.0.1" "threadcount"="10" "mysql.port"="4000" "operationcount"="0" "dotransactions"="false" ********************************************** table processor_delay.t12 not exists for 2-th check, retry later table savepoint.finish_mark not exists for 4-th check, retry later table ddl_sequence.finish_mark not exists for 5-th check, retry later table force_replicate_table.t5 not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t12 not exists for 3-th check, retry later table savepoint.finish_mark not exists for 5-th check, retry later Run finished, takes 2.665895636s INSERT - Takes(s): 2.6, Count: 9997, OPS: 3789.7, Avg(us): 2611, Min(us): 836, Max(us): 78587, 95th(us): 3000, 99th(us): 42000 [Mon May 6 19:14:16 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2387623878.out server --log-file /tmp/tidb_cdc_test/move_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table force_replicate_table.t5 exists table force_replicate_table.t6 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367235a00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5, pid:9515, start at 2024-05-06 19:14:16.596699437 +0800 CST m=+5.364693013 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:16.605 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:16.603 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:16.603 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_sequence.finish_mark not exists for 6-th check, retry later table processor_delay.t12 not exists for 4-th check, retry later table savepoint.finish_mark not exists for 6-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table force_replicate_table.t6 not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367235a00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5, pid:9515, start at 2024-05-06 19:14:16.596699437 +0800 CST m=+5.364693013 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:16.605 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:16.603 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:16.603 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367237240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-404-dw6f5, pid:9595, start at 2024-05-06 19:14:16.672971949 +0800 CST m=+5.351571131 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:16.681 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:16.649 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:16.649 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table ddl_sequence.finish_mark not exists for 7-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:14:19 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/653a8f79-7caf-4f67-ae07-9d08b7c9682b {"id":"653a8f79-7caf-4f67-ae07-9d08b7c9682b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994056} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c3801fd 653a8f79-7caf-4f67-ae07-9d08b7c9682b /tidb/cdc/default/default/upstream/7365843302525183663 {"id":7365843302525183663,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/653a8f79-7caf-4f67-ae07-9d08b7c9682b {"id":"653a8f79-7caf-4f67-ae07-9d08b7c9682b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994056} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c3801fd 653a8f79-7caf-4f67-ae07-9d08b7c9682b /tidb/cdc/default/default/upstream/7365843302525183663 {"id":7365843302525183663,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/653a8f79-7caf-4f67-ae07-9d08b7c9682b {"id":"653a8f79-7caf-4f67-ae07-9d08b7c9682b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994056} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c3801fd 653a8f79-7caf-4f67-ae07-9d08b7c9682b /tidb/cdc/default/default/upstream/7365843302525183663 {"id":7365843302525183663,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.23932.out cli changefeed create --start-ts=449575400516878337 '--sink-uri=file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-6136?protocol=canal-json&enable-tidb-extension=true' table processor_delay.t12 not exists for 5-th check, retry later Create changefeed successfully! ID: fd294123-b8f2-43cf-a500-ae74bc1fc4bd Info: {"upstream_id":7365843302525183663,"namespace":"default","id":"fd294123-b8f2-43cf-a500-ae74bc1fc4bd","sink_uri":"file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-6136?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:14:19.87991603+08:00","start_ts":449575400516878337,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575400516878337,"checkpoint_ts":449575400516878337,"checkpoint_time":"2024-05-06 19:14:11.044"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table savepoint.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... table force_replicate_table.t6 not exists for 3-th check, retry later table ddl_sequence.finish_mark not exists for 8-th check, retry later wait process cdc.test exit for 2-th time... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10978.out cli tso query --pd=http://127.0.0.1:2379 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:14:21 CST 2024] <<<<<< run test case savepoint success! >>>>>> + set +x + workdir=/tmp/tidb_cdc_test/move_table + sink_uri='file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-6136?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/move_table ++ date + echo '[Mon May 6 19:14:21 CST 2024] <<<<<< START storage consumer in move_table case >>>>>>' [Mon May 6 19:14:21 CST 2024] <<<<<< START storage consumer in move_table case >>>>>> + cd /tmp/tidb_cdc_test/move_table + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/move_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/move_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-6136?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' [Mon May 6 19:14:21 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2395823960.out server --log-file /tmp/tidb_cdc_test/move_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_delay.t12 exists table processor_delay.t13 exists table processor_delay.t14 not exists for 1-th check, retry later table ddl_sequence.finish_mark not exists for 9-th check, retry later Verifying downstream PD is started... + set +x + tso='449575403363500033 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575403363500033 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:14:23 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.1102611028.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table force_replicate_table.t6 not exists for 4-th check, retry later table processor_delay.t14 not exists for 2-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:14:24 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {UpstreamID:7365843302525183663 Namespace:default ID:fd294123-b8f2-43cf-a500-ae74bc1fc4bd SinkURI:file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-6136?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:14:19.87991603 +0800 CST StartTs:449575400516878337 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003abbb90 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575402823745543} {CheckpointTs:449575401067380743 MinTableBarrierTs:449575403754356745 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/653a8f79-7caf-4f67-ae07-9d08b7c9682b {"id":"653a8f79-7caf-4f67-ae07-9d08b7c9682b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994056} /tidb/cdc/default/__cdc_meta__/capture/6c6a3041-8885-488b-b503-8393dd2f7335 {"id":"6c6a3041-8885-488b-b503-8393dd2f7335","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994061} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c3801fd 653a8f79-7caf-4f67-ae07-9d08b7c9682b /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c38025b 6c6a3041-8885-488b-b503-8393dd2f7335 /tidb/cdc/default/default/changefeed/info/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"upstream-id":7365843302525183663,"namespace":"default","changefeed-id":"fd294123-b8f2-43cf-a500-ae74bc1fc4bd","sink-uri":"file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-6136?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:14:19.87991603+08:00","start-ts":449575400516878337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575402823745543} /tidb/cdc/default/default/changefeed/status/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":449575401067380743,"min-table-barrier-ts":449575403754356745,"admin-job-type":0} /tidb/cdc/default/default/task/position/653a8f79-7caf-4f67-ae07-9d08b7c9682b/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6c6a3041-8885-488b-b503-8393dd2f7335/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843302525183663 {"id":7365843302525183663,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {UpstreamID:7365843302525183663 Namespace:default ID:fd294123-b8f2-43cf-a500-ae74bc1fc4bd SinkURI:file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-6136?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:14:19.87991603 +0800 CST StartTs:449575400516878337 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003abbb90 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575402823745543} {CheckpointTs:449575401067380743 MinTableBarrierTs:449575403754356745 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/653a8f79-7caf-4f67-ae07-9d08b7c9682b {"id":"653a8f79-7caf-4f67-ae07-9d08b7c9682b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994056} /tidb/cdc/default/__cdc_meta__/capture/6c6a3041-8885-488b-b503-8393dd2f7335 {"id":"6c6a3041-8885-488b-b503-8393dd2f7335","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994061} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c3801fd 653a8f79-7caf-4f67-ae07-9d08b7c9682b /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c38025b 6c6a3041-8885-488b-b503-8393dd2f7335 /tidb/cdc/default/default/changefeed/info/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"upstream-id":7365843302525183663,"namespace":"default","changefeed-id":"fd294123-b8f2-43cf-a500-ae74bc1fc4bd","sink-uri":"file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-6136?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:14:19.87991603+08:00","start-ts":449575400516878337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575402823745543} /tidb/cdc/default/default/changefeed/status/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":449575401067380743,"min-table-barrier-ts":449575403754356745,"admin-job-type":0} /tidb/cdc/default/default/task/position/653a8f79-7caf-4f67-ae07-9d08b7c9682b/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6c6a3041-8885-488b-b503-8393dd2f7335/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843302525183663 {"id":7365843302525183663,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {UpstreamID:7365843302525183663 Namespace:default ID:fd294123-b8f2-43cf-a500-ae74bc1fc4bd SinkURI:file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-6136?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:14:19.87991603 +0800 CST StartTs:449575400516878337 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003abbb90 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575402823745543} {CheckpointTs:449575401067380743 MinTableBarrierTs:449575403754356745 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/653a8f79-7caf-4f67-ae07-9d08b7c9682b {"id":"653a8f79-7caf-4f67-ae07-9d08b7c9682b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994056} /tidb/cdc/default/__cdc_meta__/capture/6c6a3041-8885-488b-b503-8393dd2f7335 {"id":"6c6a3041-8885-488b-b503-8393dd2f7335","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994061} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c3801fd 653a8f79-7caf-4f67-ae07-9d08b7c9682b /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c38025b 6c6a3041-8885-488b-b503-8393dd2f7335 /tidb/cdc/default/default/changefeed/info/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"upstream-id":7365843302525183663,"namespace":"default","changefeed-id":"fd294123-b8f2-43cf-a500-ae74bc1fc4bd","sink-uri":"file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-6136?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:14:19.87991603+08:00","start-ts":449575400516878337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575402823745543} /tidb/cdc/default/default/changefeed/status/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":449575401067380743,"min-table-barrier-ts":449575403754356745,"admin-job-type":0} /tidb/cdc/default/default/task/position/653a8f79-7caf-4f67-ae07-9d08b7c9682b/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6c6a3041-8885-488b-b503-8393dd2f7335/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843302525183663 {"id":7365843302525183663,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:14:24 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2405824060.out server --log-file /tmp/tidb_cdc_test/move_table/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data3 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table force_replicate_table.t6 not exists for 5-th check, retry later table ddl_sequence.finish_mark not exists for 10-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:14:26 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18ed8e04-d4d7-4603-b91d-e6b657902d96 {"id":"18ed8e04-d4d7-4603-b91d-e6b657902d96","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994063} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c60a8d8 18ed8e04-d4d7-4603-b91d-e6b657902d96 /tidb/cdc/default/default/upstream/7365843348244129599 {"id":7365843348244129599,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18ed8e04-d4d7-4603-b91d-e6b657902d96 {"id":"18ed8e04-d4d7-4603-b91d-e6b657902d96","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994063} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c60a8d8 18ed8e04-d4d7-4603-b91d-e6b657902d96 /tidb/cdc/default/default/upstream/7365843348244129599 {"id":7365843348244129599,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18ed8e04-d4d7-4603-b91d-e6b657902d96 {"id":"18ed8e04-d4d7-4603-b91d-e6b657902d96","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994063} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c60a8d8 18ed8e04-d4d7-4603-b91d-e6b657902d96 /tidb/cdc/default/default/upstream/7365843348244129599 {"id":7365843348244129599,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.11084.out cli changefeed create --start-ts=449575403363500033 '--sink-uri=file:///tmp/tidb_cdc_test/partition_table/storage_test/ticdc-partition-table-test-9839?protocol=canal-json&enable-tidb-extension=true' table processor_delay.t14 not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 61183d43-6271-4d6d-b9f1-2625cad607f7 Info: {"upstream_id":7365843348244129599,"namespace":"default","id":"61183d43-6271-4d6d-b9f1-2625cad607f7","sink_uri":"file:///tmp/tidb_cdc_test/partition_table/storage_test/ticdc-partition-table-test-9839?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:14:26.95066756+08:00","start_ts":449575403363500033,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575403363500033,"checkpoint_ts":449575403363500033,"checkpoint_time":"2024-05-06 19:14:21.903"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table force_replicate_table.t6 exists check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 table ddl_sequence.finish_mark not exists for 11-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:14:27 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {UpstreamID:7365843302525183663 Namespace:default ID:fd294123-b8f2-43cf-a500-ae74bc1fc4bd SinkURI:file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-6136?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:14:19.87991603 +0800 CST StartTs:449575400516878337 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e2a630 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575402823745543} {CheckpointTs:449575401395060747 MinTableBarrierTs:449575404816039945 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/653a8f79-7caf-4f67-ae07-9d08b7c9682b {"id":"653a8f79-7caf-4f67-ae07-9d08b7c9682b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994056} /tidb/cdc/default/__cdc_meta__/capture/6c6a3041-8885-488b-b503-8393dd2f7335 {"id":"6c6a3041-8885-488b-b503-8393dd2f7335","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994061} /tidb/cdc/default/__cdc_meta__/capture/7cdcae32-ccc2-462c-87e5-cb2dabc40e1b {"id":"7cdcae32-ccc2-462c-87e5-cb2dabc40e1b","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994065} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c3801fd 653a8f79-7caf-4f67-ae07-9d08b7c9682b /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c38025b 6c6a3041-8885-488b-b503-8393dd2f7335 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c380285 7cdcae32-ccc2-462c-87e5-cb2dabc40e1b /tidb/cdc/default/default/changefeed/info/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"upstream-id":7365843302525183663,"namespace":"default","changefeed-id":"fd294123-b8f2-43cf-a500-ae74bc1fc4bd","sink-uri":"file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-6136?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:14:19.87991603+08:00","start-ts":449575400516878337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575402823745543} /tidb/cdc/default/default/changefeed/status/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":449575401395060747,"min-table-barrier-ts":449575404816039945,"admin-job-type":0} /tidb/cdc/default/default/task/position/653a8f79-7caf-4f67-ae07-9d08b7c9682b/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6c6a3041-8885-488b-b503-8393dd2f7335/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/7cdcae32-ccc2-462c-87e5-cb2dabc40e1b/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843302525183663 {"id":7365843302525183663,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {UpstreamID:7365843302525183663 Namespace:default ID:fd294123-b8f2-43cf-a500-ae74bc1fc4bd SinkURI:file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-6136?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:14:19.87991603 +0800 CST StartTs:449575400516878337 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e2a630 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575402823745543} {CheckpointTs:449575401395060747 MinTableBarrierTs:449575404816039945 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/653a8f79-7caf-4f67-ae07-9d08b7c9682b {"id":"653a8f79-7caf-4f67-ae07-9d08b7c9682b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994056} /tidb/cdc/default/__cdc_meta__/capture/6c6a3041-8885-488b-b503-8393dd2f7335 {"id":"6c6a3041-8885-488b-b503-8393dd2f7335","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994061} /tidb/cdc/default/__cdc_meta__/capture/7cdcae32-ccc2-462c-87e5-cb2dabc40e1b {"id":"7cdcae32-ccc2-462c-87e5-cb2dabc40e1b","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994065} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c3801fd 653a8f79-7caf-4f67-ae07-9d08b7c9682b /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c38025b 6c6a3041-8885-488b-b503-8393dd2f7335 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c380285 7cdcae32-ccc2-462c-87e5-cb2dabc40e1b /tidb/cdc/default/default/changefeed/info/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"upstream-id":7365843302525183663,"namespace":"default","changefeed-id":"fd294123-b8f2-43cf-a500-ae74bc1fc4bd","sink-uri":"file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-6136?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:14:19.87991603+08:00","start-ts":449575400516878337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575402823745543} /tidb/cdc/default/default/changefeed/status/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":449575401395060747,"min-table-barrier-ts":449575404816039945,"admin-job-type":0} /tidb/cdc/default/default/task/position/653a8f79-7caf-4f67-ae07-9d08b7c9682b/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6c6a3041-8885-488b-b503-8393dd2f7335/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/7cdcae32-ccc2-462c-87e5-cb2dabc40e1b/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843302525183663 {"id":7365843302525183663,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {UpstreamID:7365843302525183663 Namespace:default ID:fd294123-b8f2-43cf-a500-ae74bc1fc4bd SinkURI:file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-6136?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:14:19.87991603 +0800 CST StartTs:449575400516878337 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e2a630 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575402823745543} {CheckpointTs:449575401395060747 MinTableBarrierTs:449575404816039945 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/653a8f79-7caf-4f67-ae07-9d08b7c9682b {"id":"653a8f79-7caf-4f67-ae07-9d08b7c9682b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994056} /tidb/cdc/default/__cdc_meta__/capture/6c6a3041-8885-488b-b503-8393dd2f7335 {"id":"6c6a3041-8885-488b-b503-8393dd2f7335","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994061} /tidb/cdc/default/__cdc_meta__/capture/7cdcae32-ccc2-462c-87e5-cb2dabc40e1b {"id":"7cdcae32-ccc2-462c-87e5-cb2dabc40e1b","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994065} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c3801fd 653a8f79-7caf-4f67-ae07-9d08b7c9682b /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c38025b 6c6a3041-8885-488b-b503-8393dd2f7335 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c380285 7cdcae32-ccc2-462c-87e5-cb2dabc40e1b /tidb/cdc/default/default/changefeed/info/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"upstream-id":7365843302525183663,"namespace":"default","changefeed-id":"fd294123-b8f2-43cf-a500-ae74bc1fc4bd","sink-uri":"file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-6136?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:14:19.87991603+08:00","start-ts":449575400516878337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575402823745543} /tidb/cdc/default/default/changefeed/status/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":449575401395060747,"min-table-barrier-ts":449575404816039945,"admin-job-type":0} /tidb/cdc/default/default/task/position/653a8f79-7caf-4f67-ae07-9d08b7c9682b/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6c6a3041-8885-488b-b503-8393dd2f7335/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/7cdcae32-ccc2-462c-87e5-cb2dabc40e1b/fd294123-b8f2-43cf-a500-ae74bc1fc4bd {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843302525183663 {"id":7365843302525183663,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table move_table.usertable not exists for 1-th check, retry later run task successfully check_data_subset force_replicate_table.t1 127.0.0.1 4000 127.0.0.1 3306 + set +x + workdir=/tmp/tidb_cdc_test/partition_table + sink_uri='file:///tmp/tidb_cdc_test/partition_table/storage_test/ticdc-partition-table-test-9839?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/partition_table ++ date + echo '[Mon May 6 19:14:28 CST 2024] <<<<<< START storage consumer in partition_table case >>>>>>' [Mon May 6 19:14:28 CST 2024] <<<<<< START storage consumer in partition_table case >>>>>> + cd /tmp/tidb_cdc_test/partition_table + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/partition_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/partition_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/partition_table/storage_test/ticdc-partition-table-test-9839?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table processor_delay.t14 not exists for 4-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36709760001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:9583, start at 2024-05-06 19:13:50.085722356 +0800 CST m=+5.197441696 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:50.092 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:50.090 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:50.090 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36709760001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:9583, start at 2024-05-06 19:13:50.085722356 +0800 CST m=+5.197441696 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:50.092 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:50.090 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:50.090 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367098140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:9664, start at 2024-05-06 19:13:50.11866794 +0800 CST m=+5.175877516 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:50.129 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:50.085 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:50.085 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 2 times to try to start tidb cluster... run task successfully check_data_subset force_replicate_table.t2 127.0.0.1 4000 127.0.0.1 3306 run task successfully check_data_subset force_replicate_table.t3 127.0.0.1 4000 127.0.0.1 3306 table move_table.usertable not exists for 2-th check, retry later table ddl_sequence.finish_mark not exists for 12-th check, retry later table processor_delay.t14 not exists for 5-th check, retry later run task successfully check_data_subset force_replicate_table.t4 127.0.0.1 4000 127.0.0.1 3306 run task successfully check_data_subset force_replicate_table.t5 127.0.0.1 4000 127.0.0.1 3306 table ddl_sequence.finish_mark not exists for 13-th check, retry later run task successfully check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 table processor_delay.t14 exists table processor_delay.t15 exists table processor_delay.t16 not exists for 1-th check, retry later id=19,a=NULL doesn't exist in downstream table force_replicate_table.t6 run task failed 1-th time, retry later table move_table.usertable exists go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: storage... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status + CDC_BINARY=cdc.test + SINK_TYPE=storage + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status The 1 times to try to start tidb cluster... table ddl_sequence.finish_mark not exists for 14-th check, retry later table processor_delay.t16 not exists for 2-th check, retry later check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 id=19,a=NULL doesn't exist in downstream table force_replicate_table.t6 run task failed 2-th time, retry later go: downloading github.com/ardielle/ardielle-go v1.5.2 table ddl_sequence.finish_mark not exists for 15-th check, retry later table processor_delay.t16 not exists for 3-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table ddl_sequence.finish_mark not exists for 16-th check, retry later Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table processor_delay.t16 not exists for 4-th check, retry later check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 run task successfully table ddl_sequence.finish_mark not exists for 17-th check, retry later wait process cdc.test exit for 1-th time... table processor_delay.t16 not exists for 5-th check, retry later wait process cdc.test exit for 2-th time... table partition_table.t exists table partition_table.t1 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:14:41 CST 2024] <<<<<< run test case force_replicate_table success! >>>>>> table ddl_sequence.finish_mark not exists for 18-th check, retry later table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later table processor_delay.t16 exists table processor_delay.t17 exists table processor_delay.t18 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_sequence.finish_mark not exists for 19-th check, retry later table partition_table.t2 not exists for 2-th check, retry later table processor_delay.t18 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_sequence.finish_mark not exists for 20-th check, retry later table partition_table.t2 not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t18 not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.t2 not exists for 4-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36709760001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:9583, start at 2024-05-06 19:13:50.085722356 +0800 CST m=+5.197441696 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:50.092 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:50.090 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:50.090 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36709760001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:9583, start at 2024-05-06 19:13:50.085722356 +0800 CST m=+5.197441696 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:50.092 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:50.090 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:50.090 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367098140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:9664, start at 2024-05-06 19:13:50.11866794 +0800 CST m=+5.175877516 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:50.129 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:50.085 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:50.085 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table ddl_sequence.finish_mark not exists for 21-th check, retry later table processor_delay.t18 not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_sequence.finish_mark not exists for 22-th check, retry later table partition_table.t2 not exists for 5-th check, retry later table processor_delay.t18 not exists for 5-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367423d00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:17025, start at 2024-05-06 19:14:48.219455827 +0800 CST m=+5.313675609 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:48.226 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:48.230 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:48.230 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367423d00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:17025, start at 2024-05-06 19:14:48.219455827 +0800 CST m=+5.313675609 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:48.226 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:48.230 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:48.230 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36742614000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:17110, start at 2024-05-06 19:14:48.337855007 +0800 CST m=+5.379885241 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:48.344 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:48.325 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:48.325 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_sequence.finish_mark not exists for 23-th check, retry later table partition_table.t2 not exists for 6-th check, retry later table processor_delay.t18 exists table processor_delay.t19 exists table processor_delay.t20 not exists for 1-th check, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 3-th time, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 4-th time, retry later + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.18345.out cli tso query --pd=http://127.0.0.1:2379 table ddl_sequence.finish_mark not exists for 24-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } table partition_table.t2 not exists for 7-th check, retry later [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage table processor_delay.t20 not exists for 2-th check, retry later [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 5-th time, retry later [Pipeline] } + set +x + tso='449575411654328321 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575411654328321 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575411654328321 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Mon May 6 19:14:55 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.1838118383.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_sequence.finish_mark not exists for 25-th check, retry later table partition_table.t2 not exists for 8-th check, retry later check data successfully wait process cdc.test exit for 1-th time... table processor_delay.t20 not exists for 3-th check, retry later [2024/05/06 19:14:55.069 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/06 19:14:55.071 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/06 19:14:55.072 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=653a8f79-7caf-4f67-ae07-9d08b7c9682b] [2024/05/06 19:14:55.072 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/06 19:14:55.073 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/06 19:14:55.244 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=fd294123-b8f2-43cf-a500-ae74bc1fc4bd] [captureID=653a8f79-7caf-4f67-ae07-9d08b7c9682b] [processorDetail="{\"table_ids\":[106,108]}"] [2024/05/06 19:14:55.444 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=fd294123-b8f2-43cf-a500-ae74bc1fc4bd] [captureID=6c6a3041-8885-488b-b503-8393dd2f7335] [processorDetail="{\"table_ids\":[]}"] [2024/05/06 19:14:55.644 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=fd294123-b8f2-43cf-a500-ae74bc1fc4bd] [captureID=7cdcae32-ccc2-462c-87e5-cb2dabc40e1b] [processorDetail="{\"table_ids\":[]}"] [2024/05/06 19:14:55.644 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"653a8f79-7caf-4f67-ae07-9d08b7c9682b\":[{\"ID\":106,\"Changefeed\":\"fd294123-b8f2-43cf-a500-ae74bc1fc4bd\"},{\"ID\":108,\"Changefeed\":\"fd294123-b8f2-43cf-a500-ae74bc1fc4bd\"}],\"6c6a3041-8885-488b-b503-8393dd2f7335\":[],\"7cdcae32-ccc2-462c-87e5-cb2dabc40e1b\":[]}"] [2024/05/06 19:14:55.644 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=fd294123-b8f2-43cf-a500-ae74bc1fc4bd&target-cp-id=7cdcae32-ccc2-462c-87e5-cb2dabc40e1b&table-id=106"] [2024/05/06 19:14:55.694 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=106] [2024/05/06 19:14:55.694 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=fd294123-b8f2-43cf-a500-ae74bc1fc4bd&target-cp-id=7cdcae32-ccc2-462c-87e5-cb2dabc40e1b&table-id=108"] [2024/05/06 19:14:55.744 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=108] [2024/05/06 19:14:55.744 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=653a8f79-7caf-4f67-ae07-9d08b7c9682b] [targetCapture=7cdcae32-ccc2-462c-87e5-cb2dabc40e1b] table move_table.check1 exists check diff successfully table partition_table.t2 not exists for 9-th check, retry later wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table ddl_sequence.finish_mark exists check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:14:58 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/770a99e1-9822-4fa0-b547-07e761ec628a {"id":"770a99e1-9822-4fa0-b547-07e761ec628a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994095} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9cdc65da 770a99e1-9822-4fa0-b547-07e761ec628a /tidb/cdc/default/default/upstream/7365843474662653871 {"id":7365843474662653871,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/770a99e1-9822-4fa0-b547-07e761ec628a {"id":"770a99e1-9822-4fa0-b547-07e761ec628a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994095} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9cdc65da 770a99e1-9822-4fa0-b547-07e761ec628a /tidb/cdc/default/default/upstream/7365843474662653871 {"id":7365843474662653871,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/770a99e1-9822-4fa0-b547-07e761ec628a {"id":"770a99e1-9822-4fa0-b547-07e761ec628a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994095} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9cdc65da 770a99e1-9822-4fa0-b547-07e761ec628a /tidb/cdc/default/default/upstream/7365843474662653871 {"id":7365843474662653871,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575411654328321 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.18439.out cli changefeed create --start-ts=449575411654328321 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml wait process cdc.test exit for 1-th time... wait process cdc.test exit for 3-th time... Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365843474662653871,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:14:58.557715065+08:00","start_ts":449575411654328321,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575411654328321,"checkpoint_ts":449575411654328321,"checkpoint_time":"2024-05-06 19:14:53.530"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:14:58 CST 2024] <<<<<< run test case ddl_puller_lag success! >>>>>> wait process cdc.test exit for 3-th time... table processor_delay.t20 not exists for 4-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:14:59 CST 2024] <<<<<< run test case ddl_sequence success! >>>>>> + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 1460 0 --:--:-- --:--:-- --:--:-- 1463 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 19:14:59.930","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-06 19:15:00.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:14:59.930","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:15:00.000","info":"Data' syncing is 'finished"}' ++ jq .synced table partition_table.t2 not exists for 10-th check, retry later + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:14:59.930","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:15:00.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-06 19:14:59.930' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:14:59.930","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:15:00.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:14:59.930","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:15:00.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-06 19:15:00' + echo 'sink_checkpoint_ts is 2024-05-06' 19:14:59.930 sink_checkpoint_ts is 2024-05-06 19:14:59.930 ++ date -d '2024-05-06 19:14:59.930' +%s + checkpoint_timestamp=1714994099 ++ date -d '2024-05-06 19:15:00' +%s + current_timestamp=1714994100 + '[' 1 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later [2024/05/06 19:15:00.773 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/06 19:15:00.775 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/06 19:15:00.775 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=653a8f79-7caf-4f67-ae07-9d08b7c9682b] [2024/05/06 19:15:00.775 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/06 19:15:00.776 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/06 19:15:00.943 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=fd294123-b8f2-43cf-a500-ae74bc1fc4bd] [captureID=653a8f79-7caf-4f67-ae07-9d08b7c9682b] [processorDetail="{\"table_ids\":[110]}"] [2024/05/06 19:15:01.143 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=fd294123-b8f2-43cf-a500-ae74bc1fc4bd] [captureID=6c6a3041-8885-488b-b503-8393dd2f7335] [processorDetail="{\"table_ids\":[]}"] [2024/05/06 19:15:01.343 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=fd294123-b8f2-43cf-a500-ae74bc1fc4bd] [captureID=7cdcae32-ccc2-462c-87e5-cb2dabc40e1b] [processorDetail="{\"table_ids\":[108]}"] [2024/05/06 19:15:01.343 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"653a8f79-7caf-4f67-ae07-9d08b7c9682b\":[{\"ID\":110,\"Changefeed\":\"fd294123-b8f2-43cf-a500-ae74bc1fc4bd\"}],\"6c6a3041-8885-488b-b503-8393dd2f7335\":[],\"7cdcae32-ccc2-462c-87e5-cb2dabc40e1b\":[{\"ID\":108,\"Changefeed\":\"fd294123-b8f2-43cf-a500-ae74bc1fc4bd\"}]}"] [2024/05/06 19:15:01.343 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=fd294123-b8f2-43cf-a500-ae74bc1fc4bd&target-cp-id=7cdcae32-ccc2-462c-87e5-cb2dabc40e1b&table-id=110"] table processor_delay.t20 exists table processor_delay.t21 exists table processor_delay.t22 not exists for 1-th check, retry later [2024/05/06 19:15:01.394 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=110] [2024/05/06 19:15:01.394 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=653a8f79-7caf-4f67-ae07-9d08b7c9682b] [targetCapture=7cdcae32-ccc2-462c-87e5-cb2dabc40e1b] check diff failed 1-th time, retry later table partition_table.t2 not exists for 11-th check, retry later table test.t1 exists + sleep 5 check diff successfully table partition_table.t2 not exists for 12-th check, retry later table processor_delay.t22 not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table move_table.check2 not exists for 1-th check, retry later table partition_table.t2 not exists for 13-th check, retry later table processor_delay.t22 not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table move_table.check2 not exists for 2-th check, retry later table partition_table.t2 not exists for 14-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36709760001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:9583, start at 2024-05-06 19:13:50.085722356 +0800 CST m=+5.197441696 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:50.092 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:50.090 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:50.090 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36709760001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:9583, start at 2024-05-06 19:13:50.085722356 +0800 CST m=+5.197441696 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:50.092 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:50.090 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:50.090 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367098140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c73-vr8g3, pid:9664, start at 2024-05-06 19:13:50.11866794 +0800 CST m=+5.175877516 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:50.129 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:50.085 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:50.085 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 3306 ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.14773.out cli tso query --pd=http://127.0.0.1:2379 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2310 0 --:--:-- --:--:-- --:--:-- 2292 100 243 100 243 0 0 2308 0 --:--:-- --:--:-- --:--:-- 2292 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:15:06.980","puller_resolved_ts":"2024-05-06 19:15:00.330","last_synced_ts":"2024-05-06 19:15:00.830","now_ts":"2024-05-06 19:15:07.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:15:06.980","puller_resolved_ts":"2024-05-06' '19:15:00.330","last_synced_ts":"2024-05-06' '19:15:00.830","now_ts":"2024-05-06' '19:15:07.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:15:06.980","puller_resolved_ts":"2024-05-06' '19:15:00.330","last_synced_ts":"2024-05-06' '19:15:00.830","now_ts":"2024-05-06' '19:15:07.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 table processor_delay.t22 not exists for 4-th check, retry later table move_table.check2 not exists for 3-th check, retry later + set +x + tso='449575415528292353 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575415528292353 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575415528292353 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Mon May 6 19:15:09 CST 2024] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.1481214814.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later table processor_delay.t22 not exists for 5-th check, retry later table move_table.check2 not exists for 4-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table partition_table.finish_mark not exists for 2-th check, retry later table processor_delay.t22 exists table processor_delay.t23 exists table processor_delay.t24 not exists for 1-th check, retry later table move_table.check2 exists check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:15:12 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7365843235085893686 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:13:58.590151283 +0800 CST StartTs:449575395941679105 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00237c360 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575397239029768} {CheckpointTs:449575397606293511 MinTableBarrierTs:449575397606293511 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6 {"id":"6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994035} /tidb/cdc/default/__cdc_meta__/capture/8f6759cb-90e0-476c-b659-bcc84e0067cc {"id":"8f6759cb-90e0-476c-b659-bcc84e0067cc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994110} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9bffbed3 6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d2e23d2 8f6759cb-90e0-476c-b659-bcc84e0067cc /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7365843235085893686,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:13:58.590151283+08:00","start-ts":449575395941679105,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575397239029768} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":449575397606293511,"min-table-barrier-ts":449575397606293511,"admin-job-type":0} /tidb/cdc/default/default/task/position/6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/8f6759cb-90e0-476c-b659-bcc84e0067cc/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843235085893686 {"id":7365843235085893686,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7365843235085893686 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:13:58.590151283 +0800 CST StartTs:449575395941679105 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00237c360 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575397239029768} {CheckpointTs:449575397606293511 MinTableBarrierTs:449575397606293511 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6 {"id":"6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994035} /tidb/cdc/default/__cdc_meta__/capture/8f6759cb-90e0-476c-b659-bcc84e0067cc {"id":"8f6759cb-90e0-476c-b659-bcc84e0067cc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994110} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9bffbed3 6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d2e23d2 8f6759cb-90e0-476c-b659-bcc84e0067cc /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7365843235085893686,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:13:58.590151283+08:00","start-ts":449575395941679105,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575397239029768} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":449575397606293511,"min-table-barrier-ts":449575397606293511,"admin-job-type":0} /tidb/cdc/default/default/task/position/6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/8f6759cb-90e0-476c-b659-bcc84e0067cc/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843235085893686 {"id":7365843235085893686,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7365843235085893686 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:13:58.590151283 +0800 CST StartTs:449575395941679105 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00237c360 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575397239029768} {CheckpointTs:449575397606293511 MinTableBarrierTs:449575397606293511 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6 {"id":"6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994035} /tidb/cdc/default/__cdc_meta__/capture/8f6759cb-90e0-476c-b659-bcc84e0067cc {"id":"8f6759cb-90e0-476c-b659-bcc84e0067cc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994110} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9bffbed3 6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d2e23d2 8f6759cb-90e0-476c-b659-bcc84e0067cc /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7365843235085893686,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:13:58.590151283+08:00","start-ts":449575395941679105,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575397239029768} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":449575397606293511,"min-table-barrier-ts":449575397606293511,"admin-job-type":0} /tidb/cdc/default/default/task/position/6c75dd8e-5f78-4ce1-9c7e-fbc41fb6e6e6/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/8f6759cb-90e0-476c-b659-bcc84e0067cc/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843235085893686 {"id":7365843235085893686,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575415528292353 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.14867.out cli changefeed create --start-ts=449575415528292353 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 table partition_table.finish_mark not exists for 3-th check, retry later Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... wait process cdc.test exit for 3-th time... table processor_delay.t24 not exists for 2-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:15:14 CST 2024] <<<<<< run test case move_table success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table partition_table.finish_mark not exists for 4-th check, retry later Create changefeed successfully! ID: test-2 Info: {"upstream_id":7365843235085893686,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:15:15.81313516+08:00","start_ts":449575415528292353,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575415528292353,"checkpoint_ts":449575415528292353,"checkpoint_time":"2024-05-06 19:15:08.308"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table processor_delay.t24 not exists for 3-th check, retry later + set +x + run_sql 'use test; create table t2(id bigint primary key, a date); insert into t2 values(1, '\''2023-02-08'\'');' 127.0.0.1 4000 table partition_table.finish_mark not exists for 5-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_delay.t24 not exists for 4-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 6-th check, retry later table processor_delay.t24 not exists for 5-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 7-th check, retry later table processor_delay.t24 exists table processor_delay.t25 exists table processor_delay.t26 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 8-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t26 not exists for 2-th check, retry later table partition_table.finish_mark not exists for 9-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367663bc0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m, pid:6658, start at 2024-05-06 19:15:25.047068296 +0800 CST m=+5.465438930 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:25.056 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:25.039 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:25.039 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367674640003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms, pid:10147, start at 2024-05-06 19:15:26.107943509 +0800 CST m=+5.457667803 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:26.114 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:26.105 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:26.105 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/autorandom/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table processor_delay.t26 not exists for 3-th check, retry later table partition_table.finish_mark not exists for 10-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367663bc0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m, pid:6658, start at 2024-05-06 19:15:25.047068296 +0800 CST m=+5.465438930 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:25.056 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:25.039 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:25.039 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367666040005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m, pid:6734, start at 2024-05-06 19:15:25.18848069 +0800 CST m=+5.551197542 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:25.195 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:25.185 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:25.185 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367674640003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms, pid:10147, start at 2024-05-06 19:15:26.107943509 +0800 CST m=+5.457667803 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:26.114 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:26.105 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:26.105 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367675dc000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-pfn-9hdms, pid:10221, start at 2024-05-06 19:15:26.21548346 +0800 CST m=+5.497835850 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:26.225 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:26.199 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:26.199 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table processor_delay.t26 not exists for 4-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table partition_table.finish_mark not exists for 11-th check, retry later [Mon May 6 19:15:29 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.80538055.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.11602.out cli tso query --pd=http://127.0.0.1:2379 table processor_delay.t26 exists table processor_delay.t27 not exists for 1-th check, retry later table partition_table.finish_mark not exists for 12-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + set +x + tso='449575421556817921 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575421556817921 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:15:32 CST 2024] <<<<<< START cdc server in resourcecontrol case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.1163711639.out server --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resourcecontrol/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:15:33 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b7c2a80-58cf-4dc3-82db-7102c2e864fa {"id":"2b7c2a80-58cf-4dc3-82db-7102c2e864fa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994130} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d6dbfcb 2b7c2a80-58cf-4dc3-82db-7102c2e864fa /tidb/cdc/default/default/upstream/7365843643235062875 {"id":7365843643235062875,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b7c2a80-58cf-4dc3-82db-7102c2e864fa {"id":"2b7c2a80-58cf-4dc3-82db-7102c2e864fa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994130} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d6dbfcb 2b7c2a80-58cf-4dc3-82db-7102c2e864fa /tidb/cdc/default/default/upstream/7365843643235062875 {"id":7365843643235062875,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b7c2a80-58cf-4dc3-82db-7102c2e864fa {"id":"2b7c2a80-58cf-4dc3-82db-7102c2e864fa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994130} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d6dbfcb 2b7c2a80-58cf-4dc3-82db-7102c2e864fa /tidb/cdc/default/default/upstream/7365843643235062875 {"id":7365843643235062875,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.8100.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-31521?protocol=canal-json&enable-tidb-extension=true' -c=ddl-only-block-related-table Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7365843643235062875,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-31521?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:15:33.494029803+08:00","start_ts":449575422103388164,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575422103388164,"checkpoint_ts":449575422103388164,"checkpoint_time":"2024-05-06 19:15:33.390"} PASS table processor_delay.t27 exists table processor_delay.t28 not exists for 1-th check, retry later coverage: 2.5% of statements in github.com/pingcap/tiflow/... table partition_table.finish_mark not exists for 13-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + workdir=/tmp/tidb_cdc_test/ddl_only_block_related_table + sink_uri='file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-31521?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/ddl_only_block_related_table ++ date + echo '[Mon May 6 19:15:34 CST 2024] <<<<<< START storage consumer in ddl_only_block_related_table case >>>>>>' [Mon May 6 19:15:34 CST 2024] <<<<<< START storage consumer in ddl_only_block_related_table case >>>>>> + cd /tmp/tidb_cdc_test/ddl_only_block_related_table + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/ddl_only_block_related_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-31521?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table processor_delay.t28 not exists for 2-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table partition_table.finish_mark not exists for 14-th check, retry later < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:15:35 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25a3a597-d9b4-41a0-b749-b0dd3a7339a2 {"id":"25a3a597-d9b4-41a0-b749-b0dd3a7339a2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994133} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d72e2d6 25a3a597-d9b4-41a0-b749-b0dd3a7339a2 /tidb/cdc/default/default/upstream/7365843646819790831 {"id":7365843646819790831,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25a3a597-d9b4-41a0-b749-b0dd3a7339a2 {"id":"25a3a597-d9b4-41a0-b749-b0dd3a7339a2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994133} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d72e2d6 25a3a597-d9b4-41a0-b749-b0dd3a7339a2 /tidb/cdc/default/default/upstream/7365843646819790831 {"id":7365843646819790831,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25a3a597-d9b4-41a0-b749-b0dd3a7339a2 {"id":"25a3a597-d9b4-41a0-b749-b0dd3a7339a2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994133} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d72e2d6 25a3a597-d9b4-41a0-b749-b0dd3a7339a2 /tidb/cdc/default/default/upstream/7365843646819790831 {"id":7365843646819790831,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.11694.out cli changefeed create --start-ts=449575421556817921 '--sink-uri=file:///tmp/tidb_cdc_test/resourcecontrol/storage_test/ticdc-resourcecontrol-test-9743?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: 11ef8c0d-2ff5-434b-be30-3c30d7f0c75b Info: {"upstream_id":7365843646819790831,"namespace":"default","id":"11ef8c0d-2ff5-434b-be30-3c30d7f0c75b","sink_uri":"file:///tmp/tidb_cdc_test/resourcecontrol/storage_test/ticdc-resourcecontrol-test-9743?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:15:36.321557009+08:00","start_ts":449575421556817921,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575421556817921,"checkpoint_ts":449575421556817921,"checkpoint_time":"2024-05-06 19:15:31.305"} PASS Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.5% of statements in github.com/pingcap/tiflow/... table processor_delay.t28 not exists for 3-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later + set +x + workdir=/tmp/tidb_cdc_test/resourcecontrol + sink_uri='file:///tmp/tidb_cdc_test/resourcecontrol/storage_test/ticdc-resourcecontrol-test-9743?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/resourcecontrol ++ date + echo '[Mon May 6 19:15:37 CST 2024] <<<<<< START storage consumer in resourcecontrol case >>>>>>' [Mon May 6 19:15:37 CST 2024] <<<<<< START storage consumer in resourcecontrol case >>>>>> + cd /tmp/tidb_cdc_test/resourcecontrol + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/resourcecontrol + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/resourcecontrol/storage_test/ticdc-resourcecontrol-test-9743?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table resourcecontrol.finish_mark not exists for 1-th check, retry later table partition_table.finish_mark not exists for 15-th check, retry later table processor_delay.t28 not exists for 4-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table resourcecontrol.finish_mark not exists for 2-th check, retry later table processor_delay.t28 not exists for 5-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 4-th check, retry later table partition_table.finish_mark exists check diff successfully table resourcecontrol.finish_mark not exists for 3-th check, retry later wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36776d9c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c8j-cs84c, pid:29390, start at 2024-05-06 19:15:42.09031533 +0800 CST m=+5.944749702 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:42.102 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:42.105 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:42.105 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... table processor_delay.t28 exists table processor_delay.t29 exists table processor_delay.t30 not exists for 1-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 5-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:15:43 CST 2024] <<<<<< run test case partition_table success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36776d9c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c8j-cs84c, pid:29390, start at 2024-05-06 19:15:42.09031533 +0800 CST m=+5.944749702 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:42.102 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:42.105 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:42.105 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36776da40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-c8j-cs84c, pid:29468, start at 2024-05-06 19:15:42.106667392 +0800 CST m=+5.907708252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:42.118 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:42.107 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:42.107 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/autorandom/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/autorandom/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table resourcecontrol.finish_mark not exists for 4-th check, retry later table processor_delay.t30 not exists for 2-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 6-th check, retry later [Mon May 6 19:15:46 CST 2024] <<<<<< START cdc server in autorandom case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.autorandom.3070130703.out server --log-file /tmp/tidb_cdc_test/autorandom/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/autorandom/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table resourcecontrol.finish_mark not exists for 5-th check, retry later table processor_delay.t30 not exists for 3-th check, retry later table ddl_only_block_related_table.finish_mark exists wait process 8058 exit for 1-th time... table resourcecontrol.finish_mark exists check diff successfully wait process 8058 exit for 2-th time... wait process cdc.test exit for 1-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8058) - No such process wait process 8058 exit for 3-th time... process 8058 already exit [Mon May 6 19:15:49 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.82388240.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_delay.t30 not exists for 4-th check, retry later wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:15:49 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9e7686b5-3738-44b8-9833-f0dbf7e560a0 {"id":"9e7686b5-3738-44b8-9833-f0dbf7e560a0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994146} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9dac7fcf 9e7686b5-3738-44b8-9833-f0dbf7e560a0 /tidb/cdc/default/default/upstream/7365843715322249421 {"id":7365843715322249421,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9e7686b5-3738-44b8-9833-f0dbf7e560a0 {"id":"9e7686b5-3738-44b8-9833-f0dbf7e560a0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994146} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9dac7fcf 9e7686b5-3738-44b8-9833-f0dbf7e560a0 /tidb/cdc/default/default/upstream/7365843715322249421 {"id":7365843715322249421,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9e7686b5-3738-44b8-9833-f0dbf7e560a0 {"id":"9e7686b5-3738-44b8-9833-f0dbf7e560a0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994146} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9dac7fcf 9e7686b5-3738-44b8-9833-f0dbf7e560a0 /tidb/cdc/default/default/upstream/7365843715322249421 {"id":7365843715322249421,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 742ed9c4-e7b4-4e41-9f4e-7fecf73952f7 Info: {"upstream_id":7365843715322249421,"namespace":"default","id":"742ed9c4-e7b4-4e41-9f4e-7fecf73952f7","sink_uri":"file:///tmp/tidb_cdc_test/autorandom/storage_test/ticdc-autorandom-test-17964?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:15:49.727604702+08:00","start_ts":449575426354053127,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575426354053127,"checkpoint_ts":449575426354053127,"checkpoint_time":"2024-05-06 19:15:49.605"} + workdir=/tmp/tidb_cdc_test/autorandom + sink_uri='file:///tmp/tidb_cdc_test/autorandom/storage_test/ticdc-autorandom-test-17964?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/autorandom ++ date + echo '[Mon May 6 19:15:49 CST 2024] <<<<<< START storage consumer in autorandom case >>>>>>' [Mon May 6 19:15:49 CST 2024] <<<<<< START storage consumer in autorandom case >>>>>> + cd /tmp/tidb_cdc_test/autorandom + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/autorandom + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/autorandom/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/autorandom/storage_test/ticdc-autorandom-test-17964?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table autorandom_test.table_a not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:15:49 CST 2024] <<<<<< run test case resourcecontrol success! >>>>>> table processor_delay.t30 exists table processor_delay.t31 not exists for 1-th check, retry later table autorandom_test.table_a not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:15:52 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365843643235062875 Namespace:default ID:ddl-only-block-related-table SinkURI:file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-31521?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:15:33.494029803 +0800 CST StartTs:449575422103388164 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00384be60 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575422129340417} {CheckpointTs:449575425576796165 MinTableBarrierTs:449575426900361218 AdminJobType:noop} span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449575426900361218, checkpointTs: 449575426900361218, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575426900361218, checkpointTs: 449575426900361218, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449575426887516166, checkpointTs: 449575426887516166, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575426900361218, checkpointTs: 449575426900361218, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6d11a32e-fc1c-4263-ba3c-b8b50fedb122 {"id":"6d11a32e-fc1c-4263-ba3c-b8b50fedb122","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994149} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d6dc0bd 6d11a32e-fc1c-4263-ba3c-b8b50fedb122 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365843643235062875,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-31521?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:15:33.494029803+08:00","start-ts":449575422103388164,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575422129340417} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449575426887516166,"min-table-barrier-ts":449575426900361218,"admin-job-type":0} /tidb/cdc/default/default/task/position/6d11a32e-fc1c-4263-ba3c-b8b50fedb122/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843643235062875 {"id":7365843643235062875,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365843643235062875 Namespace:default ID:ddl-only-block-related-table SinkURI:file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-31521?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:15:33.494029803 +0800 CST StartTs:449575422103388164 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00384be60 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575422129340417} {CheckpointTs:449575425576796165 MinTableBarrierTs:449575426900361218 AdminJobType:noop} span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449575426900361218, checkpointTs: 449575426900361218, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575426900361218, checkpointTs: 449575426900361218, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449575426887516166, checkpointTs: 449575426887516166, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575426900361218, checkpointTs: 449575426900361218, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6d11a32e-fc1c-4263-ba3c-b8b50fedb122 {"id":"6d11a32e-fc1c-4263-ba3c-b8b50fedb122","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994149} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d6dc0bd 6d11a32e-fc1c-4263-ba3c-b8b50fedb122 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365843643235062875,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-31521?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:15:33.494029803+08:00","start-ts":449575422103388164,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575422129340417} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449575426887516166,"min-table-barrier-ts":449575426900361218,"admin-job-type":0} /tidb/cdc/default/default/task/position/6d11a32e-fc1c-4263-ba3c-b8b50fedb122/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843643235062875 {"id":7365843643235062875,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365843643235062875 Namespace:default ID:ddl-only-block-related-table SinkURI:file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-31521?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:15:33.494029803 +0800 CST StartTs:449575422103388164 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00384be60 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575422129340417} {CheckpointTs:449575425576796165 MinTableBarrierTs:449575426900361218 AdminJobType:noop} span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449575426900361218, checkpointTs: 449575426900361218, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575426900361218, checkpointTs: 449575426900361218, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449575426887516166, checkpointTs: 449575426887516166, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575426900361218, checkpointTs: 449575426900361218, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6d11a32e-fc1c-4263-ba3c-b8b50fedb122 {"id":"6d11a32e-fc1c-4263-ba3c-b8b50fedb122","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994149} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d6dc0bd 6d11a32e-fc1c-4263-ba3c-b8b50fedb122 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365843643235062875,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-31521?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:15:33.494029803+08:00","start-ts":449575422103388164,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575422129340417} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449575426887516166,"min-table-barrier-ts":449575426900361218,"admin-job-type":0} /tidb/cdc/default/default/task/position/6d11a32e-fc1c-4263-ba3c-b8b50fedb122/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843643235062875 {"id":7365843643235062875,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_ts_not_forward ddl-only-block-related-table table processor_delay.t31 exists table processor_delay.t32 not exists for 1-th check, retry later run task failed 1-th time, retry later table autorandom_test.table_a not exists for 3-th check, retry later table processor_delay.t32 not exists for 2-th check, retry later check_ts_not_forward ddl-only-block-related-table table autorandom_test.table_a not exists for 4-th check, retry later table processor_delay.t32 not exists for 3-th check, retry later table autorandom_test.table_a not exists for 5-th check, retry later table processor_delay.t32 not exists for 4-th check, retry later table autorandom_test.table_a exists check diff successfully wait process cdc.test exit for 1-th time... table processor_delay.t32 exists table processor_delay.t33 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:16:02 CST 2024] <<<<<< run test case autorandom success! >>>>>> [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table processor_delay.t33 exists table processor_delay.t34 not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } table processor_delay.t34 not exists for 2-th check, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table processor_delay.t34 not exists for 3-th check, retry later table processor_delay.t34 not exists for 4-th check, retry later table processor_delay.t34 exists table processor_delay.t35 exists table processor_delay.t36 not exists for 1-th check, retry later table processor_delay.t36 not exists for 2-th check, retry later + '[' storage == mysql ']' + stop_tidb_cluster =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/generate_column/run.sh using Sink-Type: storage... <<================= [Mon May 6 19:16:13 CST 2024] <<<<<< run test case generate_column success! >>>>>> table processor_delay.t36 not exists for 3-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } table processor_delay.t36 not exists for 4-th check, retry later [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + stop_tidb_cluster table processor_delay.t36 not exists for 5-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2724 0 --:--:-- --:--:-- --:--:-- 2728 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 19:16:06.768","puller_resolved_ts":"2024-05-06 19:15:59.768","last_synced_ts":"2024-05-06 19:13:51.118","now_ts":"2024-05-06 19:16:08.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:16:06.768","puller_resolved_ts":"2024-05-06' '19:15:59.768","last_synced_ts":"2024-05-06' '19:13:51.118","now_ts":"2024-05-06' '19:16:08.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 9616 8.2 0.0 13456668 143384 ? Sl 19:13 0:13 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 9665 5.6 0.0 13587164 138104 ? Sl 19:13 0:09 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1714994174.0938008,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003665880/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714994174.0938888,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714994174.1738603,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003a05180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714994174.1739595,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714994174.9466567,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0020d3dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714994174.9467158,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T19:16:18.670399+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fe5500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:18.691159+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000c73500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:18.775449+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011fc000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table processor_delay.t36 exists table processor_delay.t37 exists table processor_delay.t38 not exists for 1-th check, retry later table processor_delay.t38 not exists for 2-th check, retry later {"level":"warn","ts":"2024-05-06T19:16:24.670983+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fe5500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:24.692062+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000c73500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:24.776639+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011fc000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } table processor_delay.t38 not exists for 3-th check, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table processor_delay.t38 not exists for 4-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed table processor_delay.t38 not exists for 5-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:16:30.671854+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fe5500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:30.692744+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000c73500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:30.777775+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011fc000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table processor_delay.t38 exists table processor_delay.t39 exists table processor_delay.t40 not exists for 1-th check, retry later table processor_delay.t40 not exists for 2-th check, retry later table processor_delay.t40 not exists for 3-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:16:36.673801+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fe5500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:36.693001+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000c73500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:36.77918+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011fc000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table processor_delay.t40 not exists for 4-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:16:38.66285+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fe5500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-06T19:16:38.662928+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T19:16:38.68189+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000c73500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-06T19:16:38.68196+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T19:16:38.765144+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011fc000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-06T19:16:38.765203+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} table processor_delay.t40 not exists for 5-th check, retry later table processor_delay.t40 exists table processor_delay.t41 exists table processor_delay.t42 not exists for 1-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:16:42.675178+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fe5500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:42.694701+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000c73500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:42.780773+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011fc000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table processor_delay.t42 not exists for 2-th check, retry later table processor_delay.t42 not exists for 3-th check, retry later table processor_delay.t42 not exists for 4-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:16:48.676803+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fe5500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:48.695446+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000c73500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:48.782495+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011fc000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":1714994209.094958,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003665880/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714994209.0950594,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714994209.175485,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003a05180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714994209.1755443,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} table processor_delay.t42 not exists for 5-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0{"level":"warn","ts":1714994209.947407,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0020d3dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714994209.9474888,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} table processor_delay.t42 exists table processor_delay.t43 exists table processor_delay.t44 not exists for 1-th check, retry later table processor_delay.t44 not exists for 2-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:16:54.678222+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fe5500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:54.697091+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000c73500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:54.783431+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011fc000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table processor_delay.t44 not exists for 3-th check, retry later table processor_delay.t44 not exists for 4-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' ++ jq -r .error_code + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table processor_delay.t44 not exists for 5-th check, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster table processor_delay.t44 exists table processor_delay.t45 exists table processor_delay.t46 not exists for 1-th check, retry later pass check, checkpoint tso not forward after 10s run task successfully wait process 8243 exit for 1-th time... wait process 8243 exit for 2-th time... wait process 8243 exit for 3-th time... wait process 8243 exit for 4-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8243) - No such process wait process 8243 exit for 5-th time... process 8243 already exit [Mon May 6 19:16:57 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.90469048.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:17:00 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365843643235062875 Namespace:default ID:ddl-only-block-related-table SinkURI:file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-31521?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:15:33.494029803 +0800 CST StartTs:449575422103388164 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00408c750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575422129340417} {CheckpointTs:449575427044802571 MinTableBarrierTs:449575444739522564 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449575427044802571, checkpointTs: 449575427044802571, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575427044802571, checkpointTs: 449575427044802571, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449575427044802571, checkpointTs: 449575427044802571, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575427044802571, checkpointTs: 449575427044802571, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a30abb8-01d1-4d3c-84d7-ccec80b4bb48 {"id":"4a30abb8-01d1-4d3c-84d7-ccec80b4bb48","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994217} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d6dc2cc 4a30abb8-01d1-4d3c-84d7-ccec80b4bb48 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365843643235062875,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-31521?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:15:33.494029803+08:00","start-ts":449575422103388164,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575422129340417} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449575427044802571,"min-table-barrier-ts":449575444739522564,"admin-job-type":0} /tidb/cdc/default/default/task/position/4a30abb8-01d1-4d3c-84d7-ccec80b4bb48/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843643235062875 {"id":7365843643235062875,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365843643235062875 Namespace:default ID:ddl-only-block-related-table SinkURI:file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-31521?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:15:33.494029803 +0800 CST StartTs:449575422103388164 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00408c750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575422129340417} {CheckpointTs:449575427044802571 MinTableBarrierTs:449575444739522564 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449575427044802571, checkpointTs: 449575427044802571, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575427044802571, checkpointTs: 449575427044802571, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449575427044802571, checkpointTs: 449575427044802571, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575427044802571, checkpointTs: 449575427044802571, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a30abb8-01d1-4d3c-84d7-ccec80b4bb48 {"id":"4a30abb8-01d1-4d3c-84d7-ccec80b4bb48","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994217} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d6dc2cc 4a30abb8-01d1-4d3c-84d7-ccec80b4bb48 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365843643235062875,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-31521?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:15:33.494029803+08:00","start-ts":449575422103388164,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575422129340417} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449575427044802571,"min-table-barrier-ts":449575444739522564,"admin-job-type":0} /tidb/cdc/default/default/task/position/4a30abb8-01d1-4d3c-84d7-ccec80b4bb48/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843643235062875 {"id":7365843643235062875,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365843643235062875 Namespace:default ID:ddl-only-block-related-table SinkURI:file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-31521?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 19:15:33.494029803 +0800 CST StartTs:449575422103388164 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00408c750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g3c107706d Epoch:449575422129340417} {CheckpointTs:449575427044802571 MinTableBarrierTs:449575444739522564 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449575427044802571, checkpointTs: 449575427044802571, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575427044802571, checkpointTs: 449575427044802571, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449575427044802571, checkpointTs: 449575427044802571, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575427044802571, checkpointTs: 449575427044802571, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a30abb8-01d1-4d3c-84d7-ccec80b4bb48 {"id":"4a30abb8-01d1-4d3c-84d7-ccec80b4bb48","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994217} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d6dc2cc 4a30abb8-01d1-4d3c-84d7-ccec80b4bb48 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365843643235062875,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-31521?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T19:15:33.494029803+08:00","start-ts":449575422103388164,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g3c107706d","epoch":449575422129340417} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449575427044802571,"min-table-barrier-ts":449575444739522564,"admin-job-type":0} /tidb/cdc/default/default/task/position/4a30abb8-01d1-4d3c-84d7-ccec80b4bb48/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843643235062875 + grep -q 'etcd info' {"id":7365843643235062875,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check diff failed 1-th time, retry later check diff failed 2-th time, retry later table processor_delay.t46 not exists for 2-th check, retry later check diff failed 3-th time, retry later table processor_delay.t46 not exists for 3-th check, retry later check diff successfully check_ts_forward ddl-only-block-related-table table processor_delay.t46 not exists for 4-th check, retry later + run_case_with_unavailable_tikv conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory changefeed is working normally rts: 449575446574530560->449575446836674560 checkpoint: 449575446574530560->449575446836674560 run task successfully wait process cdc.test exit for 1-th time... table processor_delay.t46 not exists for 5-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:17:10 CST 2024] <<<<<< run test case ddl_only_block_related_table success! >>>>>> table processor_delay.t46 exists table processor_delay.t47 exists table processor_delay.t48 not exists for 1-th check, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table processor_delay.t48 not exists for 2-th check, retry later table processor_delay.t48 not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_delay.t48 not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t48 not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t48 exists table processor_delay.t49 exists table processor_delay.t50 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/changefeed_auto_stop/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t50 not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367da6780012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:13010, start at 2024-05-06 19:17:24.013904433 +0800 CST m=+5.384165859 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:19:24.021 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:17:23.998 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:07:23.998 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table processor_delay.t50 not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367da6780012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:13010, start at 2024-05-06 19:17:24.013904433 +0800 CST m=+5.384165859 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:19:24.021 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:17:23.998 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:07:23.998 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367da9a40005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:13078, start at 2024-05-06 19:17:24.204914677 +0800 CST m=+5.524882874 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:19:24.214 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:17:24.201 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:07:24.201 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table processor_delay.t50 not exists for 4-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2665 0 --:--:-- --:--:-- --:--:-- 2695 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 19:17:17.080","puller_resolved_ts":"2024-05-06 19:17:09.080","last_synced_ts":"2024-05-06 19:15:00.830","now_ts":"2024-05-06 19:17:18.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:17:17.080","puller_resolved_ts":"2024-05-06' '19:17:09.080","last_synced_ts":"2024-05-06' '19:15:00.830","now_ts":"2024-05-06' '19:17:18.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 16257 7.4 0.0 13521688 143880 ? Sl 19:14 0:12 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 16326 5.1 0.0 13651160 139200 ? Sl 19:14 0:08 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1714994243.258798,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0039d0000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714994243.2588701,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714994243.3734992,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002daf500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714994243.373561,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714994244.1349502,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0021fec40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714994244.1350362,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T19:17:27.972953+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001081a40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:17:27.973846+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f83500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:17:28.059659+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cbb500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.14327.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_delay.t50 not exists for 5-th check, retry later + set +x + tso='449575452487974913 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575452487974913 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575452487974913 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Mon May 6 19:17:30 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1436514367.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t50 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:17:34 CST 2024] <<<<<< run test case processor_etcd_worker_delay success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:17:34 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2252a379-2e9c-4544-a184-8a5ea60a6e1f {"id":"2252a379-2e9c-4544-a184-8a5ea60a6e1f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994251} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f3ea0d6 2252a379-2e9c-4544-a184-8a5ea60a6e1f /tidb/cdc/default/default/upstream/7365844149525845289 {"id":7365844149525845289,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2252a379-2e9c-4544-a184-8a5ea60a6e1f {"id":"2252a379-2e9c-4544-a184-8a5ea60a6e1f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994251} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f3ea0d6 2252a379-2e9c-4544-a184-8a5ea60a6e1f /tidb/cdc/default/default/upstream/7365844149525845289 {"id":7365844149525845289,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2252a379-2e9c-4544-a184-8a5ea60a6e1f {"id":"2252a379-2e9c-4544-a184-8a5ea60a6e1f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994251} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f3ea0d6 2252a379-2e9c-4544-a184-8a5ea60a6e1f /tidb/cdc/default/default/upstream/7365844149525845289 {"id":7365844149525845289,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575452487974913 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.14419.out cli changefeed create --start-ts=449575452487974913 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365844149525845289,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:17:34.574610739+08:00","start_ts":449575452487974913,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575452487974913,"checkpoint_ts":449575452487974913,"checkpoint_time":"2024-05-06 19:17:29.298"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... {"level":"warn","ts":"2024-05-06T19:17:33.974198+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001081a40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:17:33.975492+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f83500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:17:34.060672+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cbb500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367e6cdc000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m, pid:10317, start at 2024-05-06 19:17:36.705000098 +0800 CST m=+5.272225715 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:19:36.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:17:36.695 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:07:36.695 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t1 exists + sleep 5 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367e6cdc000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m, pid:10317, start at 2024-05-06 19:17:36.705000098 +0800 CST m=+5.272225715 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:19:36.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:17:36.695 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:07:36.695 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367e6d90000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-sb4-mwc8m, pid:10403, start at 2024-05-06 19:17:36.75023073 +0800 CST m=+5.268826469 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:19:36.757 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:17:36.740 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:07:36.740 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:17:39.976+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f83500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:17:39.976194+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001081a40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:17:40.06152+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cbb500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} ***************** properties ***************** "requestdistribution"="uniform" "recordcount"="20" "mysql.user"="root" "workload"="core" "mysql.host"="127.0.0.1" "readproportion"="0" "threadcount"="4" "scanproportion"="0" "readallfields"="true" "insertproportion"="0" "mysql.db"="changefeed_auto_stop_1" "operationcount"="0" "mysql.port"="4000" "updateproportion"="0" "dotransactions"="false" ********************************************** Run finished, takes 10.381599ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3044.3, Avg(us): 1802, Min(us): 992, Max(us): 5465, 95th(us): 4000, 99th(us): 6000 ***************** properties ***************** "updateproportion"="0" "mysql.host"="127.0.0.1" "insertproportion"="0" "readproportion"="0" "scanproportion"="0" "readallfields"="true" "threadcount"="4" "recordcount"="20" "dotransactions"="false" "mysql.db"="changefeed_auto_stop_2" "workload"="core" "requestdistribution"="uniform" "mysql.user"="root" "operationcount"="0" "mysql.port"="4000" ********************************************** Run finished, takes 8.81914ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3958.7, Avg(us): 1629, Min(us): 940, Max(us): 3666, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "updateproportion"="0" "operationcount"="0" "recordcount"="20" "readallfields"="true" "mysql.host"="127.0.0.1" "threadcount"="4" "mysql.port"="4000" "workload"="core" "readproportion"="0" "insertproportion"="0" "mysql.db"="changefeed_auto_stop_3" "requestdistribution"="uniform" "mysql.user"="root" "dotransactions"="false" "scanproportion"="0" ********************************************** Run finished, takes 9.065033ms INSERT - Takes(s): 0.0, Count: 20, OPS: 4233.6, Avg(us): 1734, Min(us): 996, Max(us): 4387, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "readallfields"="true" "readproportion"="0" "updateproportion"="0" "mysql.db"="changefeed_auto_stop_4" "workload"="core" "dotransactions"="false" "recordcount"="20" "threadcount"="4" "requestdistribution"="uniform" "mysql.port"="4000" "operationcount"="0" "mysql.user"="root" "mysql.host"="127.0.0.1" "scanproportion"="0" "insertproportion"="0" ********************************************** Run finished, takes 8.670253ms INSERT - Takes(s): 0.0, Count: 20, OPS: 4154.5, Avg(us): 1642, Min(us): 936, Max(us): 3900, 95th(us): 4000, 99th(us): 4000 [Mon May 6 19:17:42 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1190911911.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 12378 30.2 0.5 4461812 2265552 ? Sl 19:17 0:08 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv1 jenkins 12379 22.6 0.5 4438780 2225912 ? Sl 19:17 0:06 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv2 jenkins 12380 22.6 0.5 4426488 2199088 ? Sl 19:17 0:06 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv3 jenkins 12382 29.5 0.5 4456188 2252888 ? Sl 19:17 0:07 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2510 0 --:--:-- --:--:-- --:--:-- 2531 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:17:42.147","puller_resolved_ts":"2024-05-06 19:17:36.098","last_synced_ts":"2024-05-06 19:17:36.148","now_ts":"2024-05-06 19:17:43.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:17:42.147","puller_resolved_ts":"2024-05-06' '19:17:36.098","last_synced_ts":"2024-05-06' '19:17:36.148","now_ts":"2024-05-06' '19:17:43.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:17:42.147","puller_resolved_ts":"2024-05-06' '19:17:36.098","last_synced_ts":"2024-05-06' '19:17:36.148","now_ts":"2024-05-06' '19:17:43.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:17:45 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/65ce8cc7-cfe9-40fc-bf95-5e799f11fb47 {"id":"65ce8cc7-cfe9-40fc-bf95-5e799f11fb47","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994263} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f6ede37 65ce8cc7-cfe9-40fc-bf95-5e799f11fb47 /tidb/cdc/default/default/upstream/7365844208694332101 {"id":7365844208694332101,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/65ce8cc7-cfe9-40fc-bf95-5e799f11fb47 {"id":"65ce8cc7-cfe9-40fc-bf95-5e799f11fb47","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994263} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f6ede37 65ce8cc7-cfe9-40fc-bf95-5e799f11fb47 /tidb/cdc/default/default/upstream/7365844208694332101 {"id":7365844208694332101,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/65ce8cc7-cfe9-40fc-bf95-5e799f11fb47 {"id":"65ce8cc7-cfe9-40fc-bf95-5e799f11fb47","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994263} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f6ede37 65ce8cc7-cfe9-40fc-bf95-5e799f11fb47 /tidb/cdc/default/default/upstream/7365844208694332101 {"id":7365844208694332101,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:17:45 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedError=1*return(true);github.com/pingcap/tiflow/cdc/processor/ProcessorUpdatePositionDelaying=sleep(1000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1196811970.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:17:45.977249+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001081a40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:17:45.977517+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f83500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:17:46.063204+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cbb500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:17:47.961028+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001081a40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-06T19:17:47.961079+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T19:17:47.965689+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f83500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-06T19:17:47.96573+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T19:17:48.047901+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cbb500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-06T19:17:48.047998+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/sink_hang/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:17:48 GMT < Content-Length: 1275 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/65ce8cc7-cfe9-40fc-bf95-5e799f11fb47 {"id":"65ce8cc7-cfe9-40fc-bf95-5e799f11fb47","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994263} /tidb/cdc/default/__cdc_meta__/capture/72192936-7e1b-44e2-8c20-9a3cb1a57695 {"id":"72192936-7e1b-44e2-8c20-9a3cb1a57695","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994266} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f6ede37 65ce8cc7-cfe9-40fc-bf95-5e799f11fb47 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f6ede6a 72192936-7e1b-44e2-8c20-9a3cb1a57695 /tidb/cdc/default/default/upstream/7365844208694332101 {"id":7365844208694332101,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/65ce8cc7-cfe9-40fc-bf95-5e799f11fb47 {"id":"65ce8cc7-cfe9-40fc-bf95-5e799f11fb47","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994263} /tidb/cdc/default/__cdc_meta__/capture/72192936-7e1b-44e2-8c20-9a3cb1a57695 {"id":"72192936-7e1b-44e2-8c20-9a3cb1a57695","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994266} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f6ede37 65ce8cc7-cfe9-40fc-bf95-5e799f11fb47 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f6ede6a 72192936-7e1b-44e2-8c20-9a3cb1a57695 /tidb/cdc/default/default/upstream/7365844208694332101 {"id":7365844208694332101,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/65ce8cc7-cfe9-40fc-bf95-5e799f11fb47 {"id":"65ce8cc7-cfe9-40fc-bf95-5e799f11fb47","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994263} /tidb/cdc/default/__cdc_meta__/capture/72192936-7e1b-44e2-8c20-9a3cb1a57695 {"id":"72192936-7e1b-44e2-8c20-9a3cb1a57695","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994266} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f6ede37 65ce8cc7-cfe9-40fc-bf95-5e799f11fb47 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f6ede6a 72192936-7e1b-44e2-8c20-9a3cb1a57695 /tidb/cdc/default/default/upstream/7365844208694332101 {"id":7365844208694332101,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/changefeed_auto_stop + sink_uri='file:///tmp/tidb_cdc_test/changefeed_auto_stop/storage_test/ticdc-changefeed-auto-stop-test-32012?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/changefeed_auto_stop ++ date + echo '[Mon May 6 19:17:48 CST 2024] <<<<<< START storage consumer in changefeed_auto_stop case >>>>>>' [Mon May 6 19:17:48 CST 2024] <<<<<< START storage consumer in changefeed_auto_stop case >>>>>> + cd /tmp/tidb_cdc_test/changefeed_auto_stop + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/changefeed_auto_stop + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/changefeed_auto_stop/storage_test/ticdc-changefeed-auto-stop-test-32012?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' check_changefeed_state http://127.0.0.1:2379 6d230106-61fa-436b-902e-b125e1e9b1a7 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=6d230106-61fa-436b-902e-b125e1e9b1a7 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 6d230106-61fa-436b-902e-b125e1e9b1a7 -s + info='{ "upstream_id": 7365844208694332101, "namespace": "default", "id": "6d230106-61fa-436b-902e-b125e1e9b1a7", "state": "normal", "checkpoint_tso": 449575455764250625, "checkpoint_time": "2024-05-06 19:17:41.796", "error": null }' + echo '{ "upstream_id": 7365844208694332101, "namespace": "default", "id": "6d230106-61fa-436b-902e-b125e1e9b1a7", "state": "normal", "checkpoint_tso": 449575455764250625, "checkpoint_time": "2024-05-06 19:17:41.796", "error": null }' { "upstream_id": 7365844208694332101, "namespace": "default", "id": "6d230106-61fa-436b-902e-b125e1e9b1a7", "state": "normal", "checkpoint_tso": 449575455764250625, "checkpoint_time": "2024-05-06 19:17:41.796", "error": null } ++ echo '{' '"upstream_id":' 7365844208694332101, '"namespace":' '"default",' '"id":' '"6d230106-61fa-436b-902e-b125e1e9b1a7",' '"state":' '"normal",' '"checkpoint_tso":' 449575455764250625, '"checkpoint_time":' '"2024-05-06' '19:17:41.796",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365844208694332101, '"namespace":' '"default",' '"id":' '"6d230106-61fa-436b-902e-b125e1e9b1a7",' '"state":' '"normal",' '"checkpoint_tso":' 449575455764250625, '"checkpoint_time":' '"2024-05-06' '19:17:41.796",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table changefeed_auto_stop_1.usertable not exists for 1-th check, retry later table changefeed_auto_stop_1.usertable not exists for 2-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:17:51.977878+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001081a40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:17:51.978603+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f83500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:17:52.064635+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cbb500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} start tidb cluster in /tmp/tidb_cdc_test/sink_hang Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table changefeed_auto_stop_1.usertable not exists for 3-th check, retry later table changefeed_auto_stop_1.usertable not exists for 4-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table changefeed_auto_stop_1.usertable not exists for 5-th check, retry later Starting Upstream TiDB... 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:17:57.978768+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001081a40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:17:57.979793+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f83500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) {"level":"warn","ts":"2024-05-06T19:17:58.065031+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cbb500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0{"level":"warn","ts":1714994278.2595735,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0039d0000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714994278.2596343,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714994278.3751163,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002daf500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714994278.3751948,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} table changefeed_auto_stop_1.usertable exists table changefeed_auto_stop_2.usertable not exists for 1-th check, retry later {"level":"warn","ts":1714994279.136734,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0021fec40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714994279.1367805,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_auto_stop_2.usertable exists table changefeed_auto_stop_3.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_auto_stop_3.usertable not exists for 2-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:18:03.979968+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001081a40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:18:03.980543+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f83500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:18:04.06642+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cbb500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table changefeed_auto_stop_3.usertable not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36800bf40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35j, pid:8405, start at 2024-05-06 19:18:03.291133746 +0800 CST m=+5.284949175 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:03.297 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:03.261 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:03.261 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36800bf40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35j, pid:8405, start at 2024-05-06 19:18:03.291133746 +0800 CST m=+5.284949175 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:03.297 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:03.261 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:03.261 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36800ef8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-6jw-7r35j, pid:8485, start at 2024-05-06 19:18:03.46399353 +0800 CST m=+5.393260551 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:03.472 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:03.454 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:03.454 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_hang/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_hang/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table changefeed_auto_stop_3.usertable not exists for 4-th check, retry later [Mon May 6 19:18:07 CST 2024] <<<<<< START cdc server in sink_hang case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=2*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_hang.97169718.out server --log-file /tmp/tidb_cdc_test/sink_hang/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_hang/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table changefeed_auto_stop_3.usertable not exists for 5-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:18:10 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f370f3d-33a1-4481-912b-e2e7cab65508 {"id":"6f370f3d-33a1-4481-912b-e2e7cab65508","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9fd895cc 6f370f3d-33a1-4481-912b-e2e7cab65508 /tidb/cdc/default/default/upstream/7365844321005611292 {"id":7365844321005611292,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f370f3d-33a1-4481-912b-e2e7cab65508 {"id":"6f370f3d-33a1-4481-912b-e2e7cab65508","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9fd895cc 6f370f3d-33a1-4481-912b-e2e7cab65508 /tidb/cdc/default/default/upstream/7365844321005611292 {"id":7365844321005611292,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f370f3d-33a1-4481-912b-e2e7cab65508 {"id":"6f370f3d-33a1-4481-912b-e2e7cab65508","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9fd895cc 6f370f3d-33a1-4481-912b-e2e7cab65508 /tidb/cdc/default/default/upstream/7365844321005611292 {"id":7365844321005611292,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/sink_hang + sink_uri='file:///tmp/tidb_cdc_test/sink_hang/storage_test/ticdc-sink-hang-test-9954?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/sink_hang ++ date + echo '[Mon May 6 19:18:11 CST 2024] <<<<<< START storage consumer in sink_hang case >>>>>>' [Mon May 6 19:18:11 CST 2024] <<<<<< START storage consumer in sink_hang case >>>>>> + cd /tmp/tidb_cdc_test/sink_hang + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/sink_hang + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/sink_hang/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/sink_hang/storage_test/ticdc-sink-hang-test-9954?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' check_changefeed_status 127.0.0.1:8300 f6dadfdb-8fdd-4a15-9c50-85cfde5f8502 normal last_error null + endpoint=127.0.0.1:8300 + changefeed_id=f6dadfdb-8fdd-4a15-9c50-85cfde5f8502 + expected_state=normal + field=last_error + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/f6dadfdb-8fdd-4a15-9c50-85cfde5f8502/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 844 0 --:--:-- --:--:-- --:--:-- 851 + info='{"state":"normal","resolved_ts":449575463435894792,"checkpoint_ts":449575463435894792}' + echo '{"state":"normal","resolved_ts":449575463435894792,"checkpoint_ts":449575463435894792}' {"state":"normal","resolved_ts":449575463435894792,"checkpoint_ts":449575463435894792} ++ echo '{"state":"normal","resolved_ts":449575463435894792,"checkpoint_ts":449575463435894792}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_error ]] ++ echo '{"state":"normal","resolved_ts":449575463435894792,"checkpoint_ts":449575463435894792}' ++ jq -r .last_error.message + error_msg=null + [[ ! null =~ null ]] run task successfully check_changefeed_status 127.0.0.1:8300 f6dadfdb-8fdd-4a15-9c50-85cfde5f8502 normal last_warning null + endpoint=127.0.0.1:8300 + changefeed_id=f6dadfdb-8fdd-4a15-9c50-85cfde5f8502 + expected_state=normal + field=last_warning + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/f6dadfdb-8fdd-4a15-9c50-85cfde5f8502/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 675 0 --:--:-- --:--:-- --:--:-- 677 + info='{"state":"normal","resolved_ts":449575463435894792,"checkpoint_ts":449575463435894792}' + echo '{"state":"normal","resolved_ts":449575463435894792,"checkpoint_ts":449575463435894792}' {"state":"normal","resolved_ts":449575463435894792,"checkpoint_ts":449575463435894792} ++ echo '{"state":"normal","resolved_ts":449575463435894792,"checkpoint_ts":449575463435894792}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_warning ]] ++ echo '{"state":"normal","resolved_ts":449575463435894792,"checkpoint_ts":449575463435894792}' ++ jq -r .last_warning.message + error_msg=null + [[ ! null =~ null ]] run task successfully table sink_hang.t1 not exists for 1-th check, retry later table changefeed_auto_stop_3.usertable exists table changefeed_auto_stop_4.usertable exists check diff failed 1-th time, retry later check diff failed 2-th time, retry later table sink_hang.t1 not exists for 2-th check, retry later check diff failed 3-th time, retry later table sink_hang.t1 not exists for 3-th check, retry later + run_case_with_unavailable_tikv conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory table sink_hang.t1 not exists for 4-th check, retry later check diff failed 4-th time, retry later table sink_hang.t1 not exists for 5-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:18:21 CST 2024] <<<<<< run test case changefeed_auto_stop success! >>>>>> table sink_hang.t1 exists table sink_hang.t2 exists check diff failed 1-th time, retry later Verifying downstream PD is started... check diff failed 2-th time, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 3-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3681cab00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:19673, start at 2024-05-06 19:18:31.875918655 +0800 CST m=+5.297622962 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:31.883 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:31.852 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:31.852 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3681cab00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:19673, start at 2024-05-06 19:18:31.875918655 +0800 CST m=+5.297622962 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:31.883 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:31.852 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:31.852 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3681cc2c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:19760, start at 2024-05-06 19:18:31.953813042 +0800 CST m=+5.308744789 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:31.960 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:31.947 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:31.947 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:18:34 CST 2024] <<<<<< run test case sink_hang success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.20996.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575469764313091 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575469764313091 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575469764313091 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Mon May 6 19:18:36 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2103021032.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:18:39 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1fcd1c31-d39a-4e60-afb6-014c0f28e2fa {"id":"1fcd1c31-d39a-4e60-afb6-014c0f28e2fa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994317} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da0461fd4 1fcd1c31-d39a-4e60-afb6-014c0f28e2fa /tidb/cdc/default/default/upstream/7365844439445730400 {"id":7365844439445730400,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1fcd1c31-d39a-4e60-afb6-014c0f28e2fa {"id":"1fcd1c31-d39a-4e60-afb6-014c0f28e2fa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994317} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da0461fd4 1fcd1c31-d39a-4e60-afb6-014c0f28e2fa /tidb/cdc/default/default/upstream/7365844439445730400 {"id":7365844439445730400,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1fcd1c31-d39a-4e60-afb6-014c0f28e2fa {"id":"1fcd1c31-d39a-4e60-afb6-014c0f28e2fa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994317} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da0461fd4 1fcd1c31-d39a-4e60-afb6-014c0f28e2fa /tidb/cdc/default/default/upstream/7365844439445730400 {"id":7365844439445730400,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575469764313091 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.21086.out cli changefeed create --start-ts=449575469764313091 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365844439445730400,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:18:40.225260037+08:00","start_ts":449575469764313091,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575469764313091,"checkpoint_ts":449575469764313091,"checkpoint_time":"2024-05-06 19:18:35.202"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 19037 24.2 0.5 4432120 2217652 ? Sl 19:18 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 19038 32.5 0.5 4453108 2263540 ? Sl 19:18 0:07 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 19039 24.1 0.5 4423416 2192636 ? Sl 19:18 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 19043 31.7 0.5 4453112 2253324 ? Sl 19:18 0:07 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2979 0 --:--:-- --:--:-- --:--:-- 3000 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:18:47.652","puller_resolved_ts":"2024-05-06 19:18:41.702","last_synced_ts":"2024-05-06 19:18:41.752","now_ts":"2024-05-06 19:18:48.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:18:47.652","puller_resolved_ts":"2024-05-06' '19:18:41.702","last_synced_ts":"2024-05-06' '19:18:41.752","now_ts":"2024-05-06' '19:18:48.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:18:47.652","puller_resolved_ts":"2024-05-06' '19:18:41.702","last_synced_ts":"2024-05-06' '19:18:41.752","now_ts":"2024-05-06' '19:18:48.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 8893 0 --:--:-- --:--:-- --:--:-- 8817 100 723 100 723 0 0 8884 0 --:--:-- --:--:-- --:--:-- 8817 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:17:43.148","puller_resolved_ts":"2024-05-06 19:17:43.148","last_synced_ts":"2024-05-06 19:17:36.148","now_ts":"2024-05-06 19:19:53.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:17:43.148","puller_resolved_ts":"2024-05-06' '19:17:43.148","last_synced_ts":"2024-05-06' '19:17:36.148","now_ts":"2024-05-06' '19:19:53.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:17:43.148","puller_resolved_ts":"2024-05-06' '19:17:43.148","last_synced_ts":"2024-05-06' '19:17:36.148","now_ts":"2024-05-06' '19:19:53.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d368899880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:15598, start at 2024-05-06 19:20:23.426282119 +0800 CST m=+5.209449156 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:22:23.434 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:20:23.394 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:10:23.394 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d368899880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:15598, start at 2024-05-06 19:20:23.426282119 +0800 CST m=+5.209449156 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:22:23.434 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:20:23.394 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:10:23.394 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36889bcc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:15674, start at 2024-05-06 19:20:23.567294765 +0800 CST m=+5.297719560 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:22:23.576 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:20:23.539 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:10:23.539 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.16933.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575499555143682 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575499555143682 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575499555143682 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Mon May 6 19:20:30 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1697316975.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:20:33 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e306d52-2f57-4c1d-8b36-b68d0e5f2cf5 {"id":"5e306d52-2f57-4c1d-8b36-b68d0e5f2cf5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994430} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da1fa38d8 5e306d52-2f57-4c1d-8b36-b68d0e5f2cf5 /tidb/cdc/default/default/upstream/7365844914830778102 {"id":7365844914830778102,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e306d52-2f57-4c1d-8b36-b68d0e5f2cf5 {"id":"5e306d52-2f57-4c1d-8b36-b68d0e5f2cf5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994430} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da1fa38d8 5e306d52-2f57-4c1d-8b36-b68d0e5f2cf5 /tidb/cdc/default/default/upstream/7365844914830778102 {"id":7365844914830778102,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e306d52-2f57-4c1d-8b36-b68d0e5f2cf5 {"id":"5e306d52-2f57-4c1d-8b36-b68d0e5f2cf5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994430} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da1fa38d8 5e306d52-2f57-4c1d-8b36-b68d0e5f2cf5 /tidb/cdc/default/default/upstream/7365844914830778102 {"id":7365844914830778102,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575499555143682 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.17032.out cli changefeed create --start-ts=449575499555143682 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365844914830778102,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:20:33.835764411+08:00","start_ts":449575499555143682,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575499555143682,"checkpoint_ts":449575499555143682,"checkpoint_time":"2024-05-06 19:20:28.845"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 15598 11.0 0.0 2484148 256248 ? Sl 19:20 0:02 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1714994418210674130.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb.log jenkins 15602 3.3 0.0 2670796 193300 ? Sl 19:20 0:00 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1714994418213448761.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb_other.log jenkins 15674 12.2 0.0 2921052 275604 ? Sl 19:20 0:03 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1714994418262848607.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2279 0 --:--:-- --:--:-- --:--:-- 2292 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:20:41.245","puller_resolved_ts":"2024-05-06 19:20:35.344","last_synced_ts":"2024-05-06 19:20:35.394","now_ts":"2024-05-06 19:20:43.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:20:41.245","puller_resolved_ts":"2024-05-06' '19:20:35.344","last_synced_ts":"2024-05-06' '19:20:35.394","now_ts":"2024-05-06' '19:20:43.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:20:41.245","puller_resolved_ts":"2024-05-06' '19:20:35.344","last_synced_ts":"2024-05-06' '19:20:35.394","now_ts":"2024-05-06' '19:20:43.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 8827 0 --:--:-- --:--:-- --:--:-- 8925 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:18:48.652","puller_resolved_ts":"2024-05-06 19:18:48.652","last_synced_ts":"2024-05-06 19:18:41.752","now_ts":"2024-05-06 19:20:59.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ jq .synced ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:18:48.652","puller_resolved_ts":"2024-05-06' '19:18:48.652","last_synced_ts":"2024-05-06' '19:18:41.752","now_ts":"2024-05-06' '19:20:59.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + status=false + '[' false '!=' false ']' ++ jq -r .info ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:18:48.652","puller_resolved_ts":"2024-05-06' '19:18:48.652","last_synced_ts":"2024-05-06' '19:18:41.752","now_ts":"2024-05-06' '19:20:59.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d368c174c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:22199, start at 2024-05-06 19:21:20.596010538 +0800 CST m=+5.212067021 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:23:20.602 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:21:20.595 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:11:20.595 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d368c174c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:22199, start at 2024-05-06 19:21:20.596010538 +0800 CST m=+5.212067021 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:23:20.602 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:21:20.595 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:11:20.595 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d368c18b80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:22275, start at 2024-05-06 19:21:20.707134288 +0800 CST m=+5.255424469 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:23:20.713 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:21:20.686 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:11:20.686 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.23566.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575514012385281 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575514012385281 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575514012385281 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Mon May 6 19:21:25 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2359723599.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:21:28 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/623c73d2-7053-4cd4-aa62-746f51462feb {"id":"623c73d2-7053-4cd4-aa62-746f51462feb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994485} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da2e179d1 623c73d2-7053-4cd4-aa62-746f51462feb /tidb/cdc/default/default/upstream/7365845171471864072 {"id":7365845171471864072,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/623c73d2-7053-4cd4-aa62-746f51462feb {"id":"623c73d2-7053-4cd4-aa62-746f51462feb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994485} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da2e179d1 623c73d2-7053-4cd4-aa62-746f51462feb /tidb/cdc/default/default/upstream/7365845171471864072 {"id":7365845171471864072,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/623c73d2-7053-4cd4-aa62-746f51462feb {"id":"623c73d2-7053-4cd4-aa62-746f51462feb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994485} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da2e179d1 623c73d2-7053-4cd4-aa62-746f51462feb /tidb/cdc/default/default/upstream/7365845171471864072 {"id":7365845171471864072,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575514012385281 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.23657.out cli changefeed create --start-ts=449575514012385281 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365845171471864072,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:21:29.022746706+08:00","start_ts":449575514012385281,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575514012385281,"checkpoint_ts":449575514012385281,"checkpoint_time":"2024-05-06 19:21:23.995"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 22195 3.6 0.0 2588800 192796 ? Sl 19:21 0:00 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714994475374508138.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log jenkins 22199 12.1 0.0 2738780 233300 ? Sl 19:21 0:02 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714994475377300765.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log jenkins 22275 12.6 0.0 2419632 262008 ? Sl 19:21 0:02 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714994475444840755.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2290 0 --:--:-- --:--:-- --:--:-- 2292 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:21:37.445","puller_resolved_ts":"2024-05-06 19:21:30.494","last_synced_ts":"2024-05-06 19:21:30.994","now_ts":"2024-05-06 19:21:38.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:21:37.445","puller_resolved_ts":"2024-05-06' '19:21:30.494","last_synced_ts":"2024-05-06' '19:21:30.994","now_ts":"2024-05-06' '19:21:38.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:21:37.445","puller_resolved_ts":"2024-05-06' '19:21:30.494","last_synced_ts":"2024-05-06' '19:21:30.994","now_ts":"2024-05-06' '19:21:38.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2528 0 --:--:-- --:--:-- --:--:-- 2540 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 19:22:51.395","puller_resolved_ts":"2024-05-06 19:22:44.395","last_synced_ts":"2024-05-06 19:20:35.394","now_ts":"2024-05-06 19:22:53.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:22:51.395","puller_resolved_ts":"2024-05-06' '19:22:44.395","last_synced_ts":"2024-05-06' '19:20:35.394","now_ts":"2024-05-06' '19:22:53.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:22:51.395","puller_resolved_ts":"2024-05-06' '19:22:44.395","last_synced_ts":"2024-05-06' '19:20:35.394","now_ts":"2024-05-06' '19:22:53.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d369342680019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:18212, start at 2024-05-06 19:23:18.083185028 +0800 CST m=+5.213774548 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:25:18.093 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:23:18.093 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:13:18.093 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d369342680019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:18212, start at 2024-05-06 19:23:18.083185028 +0800 CST m=+5.213774548 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:25:18.093 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:23:18.093 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:13:18.093 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d369343180014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-bpt-0pgwr, pid:18289, start at 2024-05-06 19:23:18.107273272 +0800 CST m=+5.175588723 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:25:18.113 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:23:18.086 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:13:18.086 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status_with_redo + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.19546.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575545338068994 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575545338068994 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575545338068994 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Mon May 6 19:23:24 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1958419586.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:23:28 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/08c62fd7-25ec-41a2-a6ca-9f29296fbd24 {"id":"08c62fd7-25ec-41a2-a6ca-9f29296fbd24","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994605} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da4a473db 08c62fd7-25ec-41a2-a6ca-9f29296fbd24 /tidb/cdc/default/default/upstream/7365845666142584588 {"id":7365845666142584588,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/08c62fd7-25ec-41a2-a6ca-9f29296fbd24 {"id":"08c62fd7-25ec-41a2-a6ca-9f29296fbd24","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994605} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da4a473db 08c62fd7-25ec-41a2-a6ca-9f29296fbd24 /tidb/cdc/default/default/upstream/7365845666142584588 {"id":7365845666142584588,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/08c62fd7-25ec-41a2-a6ca-9f29296fbd24 {"id":"08c62fd7-25ec-41a2-a6ca-9f29296fbd24","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994605} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da4a473db 08c62fd7-25ec-41a2-a6ca-9f29296fbd24 /tidb/cdc/default/default/upstream/7365845666142584588 {"id":7365845666142584588,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575545338068994 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.19638.out cli changefeed create --start-ts=449575545338068994 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365845666142584588,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:23:28.512829142+08:00","start_ts":449575545338068994,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575545338068994,"checkpoint_ts":449575545338068994,"checkpoint_time":"2024-05-06 19:23:23.493"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 6729 0 --:--:-- --:--:-- --:--:-- 6757 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:23:23.493","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-06 19:23:50.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:23:23.493","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:23:50.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:23:23.493","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:23:50.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2653 0 --:--:-- --:--:-- --:--:-- 2662 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 19:23:47.594","puller_resolved_ts":"2024-05-06 19:23:39.544","last_synced_ts":"2024-05-06 19:21:30.994","now_ts":"2024-05-06 19:23:48.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:23:47.594","puller_resolved_ts":"2024-05-06' '19:23:39.544","last_synced_ts":"2024-05-06' '19:21:30.994","now_ts":"2024-05-06' '19:23:48.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:23:47.594","puller_resolved_ts":"2024-05-06' '19:23:39.544","last_synced_ts":"2024-05-06' '19:21:30.994","now_ts":"2024-05-06' '19:23:48.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + check_logs /tmp/tidb_cdc_test/synced_status_with_redo ++ date + echo '[Mon May 6 19:24:01 CST 2024] <<<<<< run test case synced_status_with_redo success! >>>>>>' [Mon May 6 19:24:01 CST 2024] <<<<<< run test case synced_status_with_redo success! >>>>>> + stop_tidb_cluster Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d369668bc0020 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:24765, start at 2024-05-06 19:24:09.691773236 +0800 CST m=+5.053218368 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:26:09.697 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:24:09.697 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:14:09.697 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d369668bc0020 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:24765, start at 2024-05-06 19:24:09.691773236 +0800 CST m=+5.053218368 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:26:09.697 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:24:09.697 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:14:09.697 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36966bb80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1816-s6f-0phxb, pid:24846, start at 2024-05-06 19:24:09.878055319 +0800 CST m=+5.188847165 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:26:09.884 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:24:09.889 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:14:09.889 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.26168.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575558380519425 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575558380519425 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575558380519425 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Mon May 6 19:24:14 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2621126213.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:24:17 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9265be8b-b500-4ea4-a84f-49879ca35149 {"id":"9265be8b-b500-4ea4-a84f-49879ca35149","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994655} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da5769ccc 9265be8b-b500-4ea4-a84f-49879ca35149 /tidb/cdc/default/default/upstream/7365845899562278209 {"id":7365845899562278209,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9265be8b-b500-4ea4-a84f-49879ca35149 {"id":"9265be8b-b500-4ea4-a84f-49879ca35149","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994655} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da5769ccc 9265be8b-b500-4ea4-a84f-49879ca35149 /tidb/cdc/default/default/upstream/7365845899562278209 {"id":7365845899562278209,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9265be8b-b500-4ea4-a84f-49879ca35149 {"id":"9265be8b-b500-4ea4-a84f-49879ca35149","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g3c107706d","git-hash":"3c107706d2612c5f422cde6480fd92425795842e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714994655} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da5769ccc 9265be8b-b500-4ea4-a84f-49879ca35149 /tidb/cdc/default/default/upstream/7365845899562278209 {"id":7365845899562278209,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575558380519425 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.26264.out cli changefeed create --start-ts=449575558380519425 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365845899562278209,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:24:18.268251369+08:00","start_ts":449575558380519425,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-25-g3c107706d","resolved_ts":449575558380519425,"checkpoint_ts":449575558380519425,"checkpoint_time":"2024-05-06 19:24:13.246"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 4635 0 --:--:-- --:--:-- --:--:-- 4664 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:24:13.246","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-06 19:24:39.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ jq .synced ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:24:13.246","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:24:39.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:24:13.246","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:24:39.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + check_logs /tmp/tidb_cdc_test/synced_status ++ date + echo '[Mon May 6 19:24:50 CST 2024] <<<<<< run test case synced_status success! >>>>>>' [Mon May 6 19:24:50 CST 2024] <<<<<< run test case synced_status success! >>>>>> + stop_tidb_cluster \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1816/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline Finished: SUCCESS