Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/latest/pull_cdc_integration_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-3l4b8-2vrjv-w4wx4 Agent pingcap-tiflow-pull-cdc-integration-test-1779-3l4b8-2vrjv-w4wx4 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-3l4b8-2vrjv --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "0412ebf09328ca6176bd0acbb9b3cbcd0456fd44" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-3l4b8" name: "pingcap-tiflow-pull-cdc-integration-test-1779-3l4b8-2vrjv-w4wx4" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-3l4b8-2vrjv-w4wx4" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-3l4b8-2vrjv-w4wx4" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1779-3l4b8-2vrjv-w4wx4 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-list --no-walk 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=f461d564-6d43-43a2-a3ca-e8924e3fc655 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-3l4b8-2vrjv-w4wx4 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-3l4b8 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-ff467ea3dec646dcb34a434b6d5ddeaf65f47c579492826166e1422a0f0cf5b0 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1779-3l4b8 pingcap-tiflow-pull-cdc-integration-test-1779-3l4b8-2vrjv-w4wx4 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-3l4b8-2vrjv-w4wx4 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.6' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build3518562324=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-test-1779-3l4b8-2vrjv-w4wx4 bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-test-1779-3l4b8-2vrjv-w4wx4 bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 64662 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: d5234f4921e7437f (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 23 IN A 20.205.243.166 ;; Query time: 1 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Mon May 06 05:05:46 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11021/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11021/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11021/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11021/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [cdc/processor/processor.go, cdc/processor/sinkmanager/manager.go, cdc/processor/sinkmanager/manager_test_helper.go, pkg/sink/mysql/config.go] [Pipeline] echo diff file not matched: cdc/processor/processor.go [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-be15534-d72d260) 203677696 bytes in 2.15 secs (94829317 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/.git/ .git HEAD is now at d72d260c0 sink (ticdc): fix sourceID error POST git-upload-pack (656 bytes) From https://github.com/pingcap/tiflow = [up to date] master -> origin/master = [up to date] refs/pull/11021/head -> origin/pr/11021/head Previous HEAD position was d72d260c0 sink (ticdc): fix sourceID error HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) ๐Ÿšง Checkouting to base SHA:be1553484fe4c03594eabb8d7435c694e5fd7224... HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: be1553484fe4c03594eabb8d7435c694e5fd7224 be1553484 codec(ticdc): avro simplify the unit test (#11010) 2a7a65c6f Support Sequences (#10203) 36e9e1bf6 cli(ticdc): allow client authentication to be enabled without tls (#11005) ๐Ÿšง Pre-merge heads of pull requests to base SHA: be1553484fe4c03594eabb8d7435c694e5fd7224 ... Updating be1553484..d72d260c0 Fast-forward cdc/processor/processor.go | 2 +- cdc/processor/sinkmanager/manager.go | 33 +++++++++++++----------- cdc/processor/sinkmanager/manager_test_helper.go | 6 +++-- pkg/sink/mysql/config.go | 2 +- 4 files changed, 24 insertions(+), 19 deletions(-) ๐Ÿงพ Pre-merged result: d72d260c017170db94deda602246fcddfea2a4a1 d72d260c0 sink (ticdc): fix sourceID error be1553484 codec(ticdc): avro simplify the unit test (#11010) 2a7a65c6f Support Sequences (#10203) โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-be15534-d72d260 already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./scripts/download-integration-test-binaries.sh master Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 438 0 --:--:-- --:--:-- --:--:-- 436 100 41 100 41 0 0 437 0 --:--:-- --:--:-- --:--:-- 436 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 2210 0 --:--:-- --:--:-- --:--:-- 2277 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 2141 0 --:--:-- --:--:-- --:--:-- 2277 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 682 0 --:--:-- --:--:-- --:--:-- 694 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/06ee59bd9c683757f75fdd3469f37f50988a1a2f/centos7/tidb-server.tar.gz 2024-05-06 13:06:14 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/06ee59bd9c683757f75fdd3469f37f50988a1a2f/centos7/tidb-server.tar.gz [536606546/536606546] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz 2024-05-06 13:06:32 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz [187372022/187372022] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/72a0fd5b00235a7c56014b77ddd933e2a0d33c88/centos7/tikv-server.tar.gz 2024-05-06 13:07:16 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/72a0fd5b00235a7c56014b77ddd933e2a0d33c88/centos7/tikv-server.tar.gz [919098782/919098782] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e50de84e6d6ecdcc108990217b70b6bb3f50271/centos7/tiflash.tar.gz 2024-05-06 13:07:36 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e50de84e6d6ecdcc108990217b70b6bb3f50271/centos7/tiflash.tar.gz [456051121/456051121] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-06 13:07:42 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-05-06 13:07:44 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-05-06 13:07:44 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-05-06 13:07:45 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz 2024-05-06 13:07:51 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz [79877126/79877126] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> download schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2024-05-06 13:08:06 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 6 13:08 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 13:08 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 6 13:07 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 6 13:07 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 6 12:44 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 6 12:44 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 6 12:44 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 6 12:44 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 6 12:05 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M May 6 12:45 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 6 11:44 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M May 6 12:44 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M Apr 30 11:29 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/pd-ut ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ../tiflow/bin/xprog ./bin/ + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 6 13:08 . drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 13:08 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 6 13:07 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 6 13:07 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 6 12:44 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 6 12:44 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 6 12:44 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 6 12:44 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 6 12:05 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M May 6 12:45 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 6 11:44 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M May 6 12:44 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M Apr 30 11:29 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + ./bin/tidb-server -V Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 + ./bin/tikv-server -V TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.4.0 Git Commit Hash: d671b0840063bc2532941f02e02e12627402844c Git Branch: heads/refs/tags/v7.4.0 UTC Build Time: 2023-09-22 03:51:56 Go Version: go1.21.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x. 2 jenkins jenkins 4.0K May 6 13:08 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 13:08 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-gd72d260c0" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 05:08:11" -X "github.com/pingcap/tiflow/pkg/version.GitHash=d72d260c017170db94deda602246fcddfea2a4a1" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-gd72d260c0"' -o bin/cdc ./cmd/cdc go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/YangKeao/seahash v0.0.0-20240229041150-e7bf269c3140 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/swaggo/swag v1.16.3 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/go-openapi/spec v0.21.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/glebarez/sqlite v1.7.0 go: downloading gorm.io/driver/mysql v1.3.3 go: downloading github.com/jinzhu/now v1.1.5 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-openapi/jsonpointer v0.21.0 go: downloading github.com/go-openapi/jsonreference v0.21.0 go: downloading github.com/go-openapi/swag v0.23.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/glebarez/go-sqlite v1.21.2 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/kr/pretty v0.3.1 go: downloading modernc.org/libc v1.37.1 go: downloading modernc.org/sqlite v1.27.0 go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading k8s.io/apimachinery v0.28.6 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading modernc.org/mathutil v1.6.0 go: downloading modernc.org/memory v1.7.2 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + '[' -f ./bin/cdc_kafka_consumer ']' + make kafka_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-gd72d260c0" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 05:12:16" -X "github.com/pingcap/tiflow/pkg/version.GitHash=d72d260c017170db94deda602246fcddfea2a4a1" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-gd72d260c0"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go + '[' -f ./bin/cdc_storage_consumer ']' + make storage_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-gd72d260c0" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 05:12:28" -X "github.com/pingcap/tiflow/pkg/version.GitHash=d72d260c017170db94deda602246fcddfea2a4a1" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-gd72d260c0"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-gd72d260c0" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 05:12:40" -X "github.com/pingcap/tiflow/pkg/version.GitHash=d72d260c017170db94deda602246fcddfea2a4a1" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-gd72d260c0"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-gd72d260c0" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 05:12:40" -X "github.com/pingcap/tiflow/pkg/version.GitHash=d72d260c017170db94deda602246fcddfea2a4a1" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-gd72d260c0"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-gd72d260c0" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 05:12:40" -X "github.com/pingcap/tiflow/pkg/version.GitHash=d72d260c017170db94deda602246fcddfea2a4a1" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-gd72d260c0"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-gd72d260c0" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 05:12:40" -X "github.com/pingcap/tiflow/pkg/version.GitHash=d72d260c017170db94deda602246fcddfea2a4a1" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-gd72d260c0"' -o bin/oauth2-server ./cmd/oauth2-server/main.go go: downloading github.com/go-oauth2/oauth2/v4 v4.5.2 go: downloading github.com/tidwall/buntdb v1.3.0 go: downloading github.com/tidwall/rtred v0.1.2 go: downloading github.com/tidwall/gjson v1.14.3 go: downloading github.com/tidwall/grect v0.1.4 go: downloading github.com/tidwall/match v1.1.1 go: downloading github.com/tidwall/tinyqueue v0.1.1 go: downloading github.com/tidwall/pretty v1.2.0 $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20231213065948-336e064b488d go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading go.uber.org/dig v1.13.0 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading go.uber.org/goleak v1.3.0 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-gd72d260c0" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 05:12:40" -X "github.com/pingcap/tiflow/pkg/version.GitHash=d72d260c017170db94deda602246fcddfea2a4a1" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-gd72d260c0"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-gd72d260c0" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 05:12:40" -X "github.com/pingcap/tiflow/pkg/version.GitHash=d72d260c017170db94deda602246fcddfea2a4a1" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-gd72d260c0"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.2G drwxr-sr-x. 2 jenkins jenkins 4.0K May 6 13:20 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 13:08 .. -rwxr-xr-x. 1 jenkins jenkins 220M May 6 13:20 cdc -rwxr-xr-x. 1 jenkins jenkins 363M May 6 13:20 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 6 13:13 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 183M May 6 13:13 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 6 13:12 cdc_storage_consumer -rwxr-xr-x. 1 jenkins jenkins 12M May 6 13:13 oauth2-server + ./bin/cdc version Release Version: v8.2.0-alpha-22-gd72d260c0 Git Commit Hash: d72d260c017170db94deda602246fcddfea2a4a1 Git Branch: HEAD UTC Build Time: 2024-05-06 05:12:40 Go Version: go version go1.21.6 linux/amd64 Failpoint Build: true [Pipeline] } Cache not saved (binary/pingcap/tiflow/cdc-integration-test/rev-be15534-d72d260 already exists) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 3.0G drwxr-sr-x. 6 jenkins jenkins 4.0K May 6 13:21 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 13:08 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 6 13:20 bin -rwxr-xr-x. 1 jenkins jenkins 220M May 6 13:20 cdc -rwxr-xr-x. 1 jenkins jenkins 363M May 6 13:20 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 6 13:13 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 183M May 6 13:13 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 6 13:12 cdc_storage_consumer drwxr-sr-x. 4 jenkins jenkins 4.0K May 6 13:20 etc -rwxr-xr-x. 1 jenkins jenkins 17M May 6 13:20 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 6 13:20 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 6 13:20 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 13:20 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 6 13:20 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K May 6 13:20 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 6 13:20 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K May 6 13:20 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 6 13:20 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 6 13:20 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 6 13:20 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M May 6 13:20 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M May 6 13:20 minio -rwxr-xr-x. 1 jenkins jenkins 12M May 6 13:13 oauth2-server -rwxr-xr-x. 1 jenkins jenkins 37M May 6 13:20 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M May 6 13:20 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M May 6 13:20 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M May 6 13:20 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M May 6 13:20 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M May 6 13:20 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M May 6 13:20 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M May 6 13:20 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 6 13:20 share -rwxr-xr-x. 1 jenkins jenkins 32M May 6 13:20 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M May 6 13:20 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 6 13:20 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M May 6 13:20 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M May 6 13:21 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M May 6 13:21 xprog [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 164.41 secs (22687339 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G18') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G19') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G20') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G21') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G18') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G19') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G20') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G21') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-5vlz1-83nq6-835n1 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv [Pipeline] node Agent pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-k2zxd-03tw6 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "b43e4be1d5c7c50585d49b541c6d30c2a13746f4" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-k2zxd" name: "pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-xwksv-n7b01-mfk0q Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@392aba10; decorates RemoteLauncher[hudson.remoting.Channel@6a242957:JNLP4-connect connection from 10.233.69.128/10.233.69.128:58912] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] withEnv [Pipeline] { Agent pingcap-tiflow-pull-cdc-integration-test-1779-5vlz1-83nq6-835n1 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-5vlz1-83nq6 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "b5ccf9818fc2654f729a8228e9d78a47398653ee" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-5vlz1" name: "pingcap-tiflow-pull-cdc-integration-test-1779-5vlz1-83nq6-835n1" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-5vlz1-83nq6-835n1" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-5vlz1-83nq6-835n1" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1779-5vlz1-83nq6-835n1 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test [Pipeline] { [Pipeline] container > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] { [Pipeline] checkout The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Agent pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-60ww2-k02p4 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "b253c31b59deb57da676266b058fbaba11f26d07" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-60ww2" name: "pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-5155r-hgjhp-wsj0l No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7bd33bc8; decorates RemoteLauncher[hudson.remoting.Channel@4c23af86:JNLP4-connect connection from 10.233.126.158/10.233.126.158:60618] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-mlls2-941gl --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "08d4f559e611c0635f41c31e97015d711720c431" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-mlls2" name: "pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-test-1779-xwksv-n7b01-mfk0q is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-xwksv-n7b01 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "b8e3012d245fd333c58d1b983727148f8d5ab148" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-xwksv" name: "pingcap-tiflow-pull-cdc-integration-test-1779-xwksv-n7b01-mfk0q" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-xwksv-n7b01-mfk0q" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-xwksv-n7b01-mfk0q" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1779-xwksv-n7b01-mfk0q in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Agent pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-rqdbn-zml25 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "f95ea505b7b00f6429c8ab5f70249c201b33d220" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-rqdbn" name: "pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Still waiting to schedule task โ€˜pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkzโ€™ is offline Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-36hxm-r68df --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "29f5a7c7a742bec69dfd07677082d5cfc7cab679" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-36hxm" name: "pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Still waiting to schedule task โ€˜pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3โ€™ is offline Running on pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-m22f9-nc6v2 Agent pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-1lcp8-4b2m9 --- apiVersion: "v1" kind: "Pod" metadata: annotations: --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "4e90db388cee222e8f730da0bca6f23b66eb21df" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-m22f9" jenkins/label-digest: "d9725d1f3aa181b48f8b99ec4120e1d15f7266c2" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-1lcp8" name: "pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: name: "pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" memory: "12Gi" cpu: "4" tty: true volumeMounts: name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: resources: limits: memory: "256Mi" cpu: "100m" tty: true - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - name: "JENKINS_SECRET" - env: - name: "JENKINS_SECRET" value: "********" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd" - name: "JENKINS_AGENT_NAME" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd" value: "pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn" - name: "JENKINS_NAME" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" - emptyDir: medium: "" name: "workspace-volume" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Running on pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-2dpm3-xs70c --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "049466778cdbc5f8f76ed0bad56f958fa6039d27" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-2dpm3" name: "pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-kkw9l-ghxcj --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "cd44af4185a951aaf661731fbfd145897205e123" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-kkw9l" name: "pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Agent pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-c7kfw-mclzs --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7b8a052e004cbdb543366a7c7717e4b41e625f31" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-c7kfw" name: "pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Agent pingcap-tiflow-pull-cdc-integration-test-1779-5155r-hgjhp-wsj0l is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-5155r-hgjhp --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "cbe5d65c67eefc72560b578256f0d4d9937f1f31" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-5155r" name: "pingcap-tiflow-pull-cdc-integration-test-1779-5155r-hgjhp-wsj0l" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-5155r-hgjhp-wsj0l" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-5155r-hgjhp-wsj0l" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1779-5155r-hgjhp-wsj0l in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 30.76 secs (121274004 bytes/sec) [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G00 [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] { [Pipeline] { The recommended git tool is: git [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout Run cases: bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_suicide server_config_compatibility kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro debezium lossy_ddl storage_csv_update PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=2eb92971-5c45-46a1-84e9-176dedd013dd BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-k2zxd GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1779-k2zxd pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... The recommended git tool is: git [Pipeline] checkout [Pipeline] { The recommended git tool is: git [Pipeline] { The recommended git tool is: git [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-st1k9-fprpv-trhhs Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-dnsg9-wwwnp-xssfm [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-63kf5-dnt52-vw515 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-l2sg4-3h871-3slfr Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-5gqnj-hmhkq-wjjv4 [Pipeline] withCredentials Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-wn6pn-dr756-hx56v Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-r8vbf-lj78v-9fw4x [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-855t7-fbskf-4lwp5 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1779-0cr08-7wrc3-1z3gv [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6d6ca21a; decorates RemoteLauncher[hudson.remoting.Channel@4545ef60:JNLP4-connect connection from 10.233.72.134/10.233.72.134:57808] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@743b2f96; decorates RemoteLauncher[hudson.remoting.Channel@65dffa40:JNLP4-connect connection from 10.233.108.145/10.233.108.145:47244] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6825938d; decorates RemoteLauncher[hudson.remoting.Channel@5dc47646:JNLP4-connect connection from 10.233.71.227/10.233.71.227:34580] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5fd06ce8; decorates RemoteLauncher[hudson.remoting.Channel@4ebce912:JNLP4-connect connection from 10.233.88.229/10.233.88.229:47418] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@68bd329c; decorates RemoteLauncher[hudson.remoting.Channel@789e2e3d:JNLP4-connect connection from 10.233.106.99/10.233.106.99:50834] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7164ea75; decorates RemoteLauncher[hudson.remoting.Channel@e1b530e:JNLP4-connect connection from 10.233.66.124/10.233.66.124:49502] will be ignored (a typical symptom is the Git executable not being run inside a designated container) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@640e8023; decorates RemoteLauncher[hudson.remoting.Channel@17633478:JNLP4-connect connection from 10.233.69.9/10.233.69.9:54854] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@509cf0d0; decorates RemoteLauncher[hudson.remoting.Channel@27e89106:JNLP4-connect connection from 10.233.88.114/10.233.88.114:53362] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1b2c56ca; decorates RemoteLauncher[hudson.remoting.Channel@15af719c:JNLP4-connect connection from 10.233.93.139/10.233.93.139:42102] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4a797f9; decorates RemoteLauncher[hudson.remoting.Channel@317f9f8d:JNLP4-connect connection from 10.233.106.165/10.233.106.165:47540] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@60d3a20d; decorates RemoteLauncher[hudson.remoting.Channel@754c47bc:JNLP4-connect connection from 10.233.68.155/10.233.68.155:53766] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) start tidb cluster in /tmp/tidb_cdc_test/bdr_mode Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent pingcap-tiflow-pull-cdc-integration-test-1779-st1k9-fprpv-trhhs is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-st1k9-fprpv Agent pingcap-tiflow-pull-cdc-integration-test-1779-855t7-fbskf-4lwp5 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-855t7-fbskf --- apiVersion: "v1" kind: "Pod" metadata: annotations: --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" jenkins/label-digest: "06eb88ad95bbf039471116eb1c62fb57b7ec5de9" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-st1k9" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" name: "pingcap-tiflow-pull-cdc-integration-test-1779-st1k9-fprpv-trhhs" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: jenkins/label-digest: "4406f00d2e63dc49b9d049699ea8e3651a30dfb0" requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-855t7" operator: "In" values: - "amd64" containers: name: "pingcap-tiflow-pull-cdc-integration-test-1779-855t7-fbskf-4lwp5" namespace: "jenkins-tiflow" spec: affinity: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" memory: "16Gi" cpu: "6" requests: memory: "12Gi" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "report" resources: limits: memory: "256Mi" cpu: "100m" name: "workspace-volume" readOnly: false tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: readOnly: false - env: - name: "JENKINS_SECRET" value: "********" memory: "128Mi" cpu: "100m" tty: true volumeMounts: - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false value: "pingcap-tiflow-pull-cdc-integration-test-1779-st1k9-fprpv-trhhs" - name: "JENKINS_NAME" - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" value: "pingcap-tiflow-pull-cdc-integration-test-1779-st1k9-fprpv-trhhs" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" resources: limits: memory: "256Mi" cpu: "100m" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-855t7-fbskf-4lwp5" volumes: - emptyDir: medium: "" name: "workspace-volume" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-855t7-fbskf-4lwp5" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1779-855t7-fbskf-4lwp5 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Running on pingcap-tiflow-pull-cdc-integration-test-1779-st1k9-fprpv-trhhs in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Agent pingcap-tiflow-pull-cdc-integration-test-1779-wn6pn-dr756-hx56v is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-wn6pn-dr756 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "79d6153da63fbbb706f81fadfcabd19b09179ead" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-wn6pn" name: "pingcap-tiflow-pull-cdc-integration-test-1779-wn6pn-dr756-hx56v" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-wn6pn-dr756-hx56v" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-wn6pn-dr756-hx56v" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Running on pingcap-tiflow-pull-cdc-integration-test-1779-wn6pn-dr756-hx56v in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent pingcap-tiflow-pull-cdc-integration-test-1779-dnsg9-wwwnp-xssfm is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-dnsg9-wwwnp --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "01daa013bc23d14209ef810938e1ff3d33f32af6" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-dnsg9" name: "pingcap-tiflow-pull-cdc-integration-test-1779-dnsg9-wwwnp-xssfm" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-dnsg9-wwwnp-xssfm" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-dnsg9-wwwnp-xssfm" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1779-dnsg9-wwwnp-xssfm in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Agent pingcap-tiflow-pull-cdc-integration-test-1779-63kf5-dnt52-vw515 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-63kf5-dnt52 Agent pingcap-tiflow-pull-cdc-integration-test-1779-r8vbf-lj78v-9fw4x is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-r8vbf-lj78v --- apiVersion: "v1" kind: "Pod" metadata: annotations: --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/label-digest: "2f60b509da0d608183a7fa56a89c4022e9fa3738" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-r8vbf" jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "cbfdd30e67a49fb12e5dce391c2918ef189df33a" name: "pingcap-tiflow-pull-cdc-integration-test-1779-r8vbf-lj78v-9fw4x" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-63kf5" name: "pingcap-tiflow-pull-cdc-integration-test-1779-63kf5-dnt52-vw515" requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" - key: "kubernetes.io/arch" operator: "In" values: - "amd64" operator: "In" values: - "amd64" containers: containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" resources: limits: memory: "16Gi" cpu: "6" requests: cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true memory: "12Gi" cpu: "4" tty: true volumeMounts: volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" memory: "128Mi" cpu: "100m" tty: true volumeMounts: name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false resources: limits: memory: "256Mi" cpu: "100m" tty: true - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" value: "pingcap-tiflow-pull-cdc-integration-test-1779-63kf5-dnt52-vw515" - name: "JENKINS_NAME" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-r8vbf-lj78v-9fw4x" value: "pingcap-tiflow-pull-cdc-integration-test-1779-63kf5-dnt52-vw515" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-r8vbf-lj78v-9fw4x" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" requests: memory: "256Mi" cpu: "100m" volumeMounts: value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" requests: memory: "256Mi" cpu: "100m" volumeMounts: securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" name: "workspace-volume" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1779-63kf5-dnt52-vw515 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Running on pingcap-tiflow-pull-cdc-integration-test-1779-r8vbf-lj78v-9fw4x in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-test-1779-0cr08-7wrc3-1z3gv is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-0cr08-7wrc3 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "be423cddb69b31ad9cf8efc6ed8e94daa9ac7f85" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-0cr08" name: "pingcap-tiflow-pull-cdc-integration-test-1779-0cr08-7wrc3-1z3gv" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-0cr08-7wrc3-1z3gv" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1779-0cr08-7wrc3-1z3gv" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1779-0cr08-7wrc3-1z3gv in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Agent pingcap-tiflow-pull-cdc-integration-test-1779-l2sg4-3h871-3slfr is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-l2sg4-3h871 Agent pingcap-tiflow-pull-cdc-integration-test-1779-5gqnj-hmhkq-wjjv4 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1779-5gqnj-hmhkq --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" labels: jenkins/jenkins-jenkins-agent: "true" --- apiVersion: "v1" kind: "Pod" metadata: annotations: jenkins/label-digest: "da676555c6aedc1cc8f9cc12917e68869646b013" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-5gqnj" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/" name: "pingcap-tiflow-pull-cdc-integration-test-1779-5gqnj-hmhkq-wjjv4" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: labels: jenkins/jenkins-jenkins-agent: "true" requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: jenkins/label-digest: "07db5e23ba65702a00a61a5d5f40f9a6a75016eb" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1779-l2sg4" - key: "kubernetes.io/arch" operator: "In" values: - "amd64" name: "pingcap-tiflow-pull-cdc-integration-test-1779-l2sg4-3h871-3slfr" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: resources: limits: memory: "16Gi" cpu: "6" requests: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" memory: "12Gi" cpu: "4" tty: true volumeMounts: containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false resources: limits: memory: "16Gi" cpu: "6" requests: - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" memory: "12Gi" cpu: "4" tty: true volumeMounts: cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" name: "report" resources: limits: memory: "256Mi" cpu: "100m" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" name: "report" resources: limits: memory: "256Mi" cpu: "100m" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" value: "pingcap-tiflow-pull-cdc-integration-test-1779-5gqnj-hmhkq-wjjv4" - name: "JENKINS_NAME" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" value: "pingcap-tiflow-pull-cdc-integration-test-1779-5gqnj-hmhkq-wjjv4" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" value: "pingcap-tiflow-pull-cdc-integration-test-1779-l2sg4-3h871-3slfr" - name: "JENKINS_NAME" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: value: "pingcap-tiflow-pull-cdc-integration-test-1779-l2sg4-3h871-3slfr" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1779-5gqnj-hmhkq-wjjv4 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Running on pingcap-tiflow-pull-cdc-integration-test-1779-l2sg4-3h871-3slfr in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31729bd80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d, pid:1165, start at 2024-05-06 13:24:51.612840323 +0800 CST m=+5.781449012 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:26:51.621 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:24:51.622 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:14:51.622 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31729bd80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d, pid:1165, start at 2024-05-06 13:24:51.612840323 +0800 CST m=+5.781449012 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:26:51.621 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:24:51.622 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:14:51.622 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31729d6c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d, pid:1237, start at 2024-05-06 13:24:51.698987673 +0800 CST m=+5.792358515 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:26:51.709 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:24:51.675 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:14:51.675 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 15.96 secs (233749035 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G05 Run cases: charset_gbk ddl_manager multi_source PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=e218218d-8cad-4417-b1f0-a684ee805833 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-5vlz1-83nq6-835n1 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-5vlz1 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1779-5vlz1-83nq6-835n1 pingcap_tiflow_pull_cdc_integration_test_1779-5vlz1 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-5vlz1-83nq6-835n1 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Mon May 6 13:24:56 CST 2024] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.25882590.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] withCredentials No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7d9bc66b; decorates RemoteLauncher[hudson.remoting.Channel@5348890d:JNLP4-connect connection from 10.233.127.82/10.233.127.82:33690] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@37eb56c9; decorates RemoteLauncher[hudson.remoting.Channel@34999362:JNLP4-connect connection from 10.233.97.244/10.233.97.244:52118] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@339e78c1; decorates RemoteLauncher[hudson.remoting.Channel@2a489126:JNLP4-connect connection from 10.233.107.205/10.233.107.205:50170] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@76cf9260; decorates RemoteLauncher[hudson.remoting.Channel@17b6a483:JNLP4-connect connection from 10.233.105.33/10.233.105.33:39816] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3dae6232; decorates RemoteLauncher[hudson.remoting.Channel@6d4ccafa:JNLP4-connect connection from 10.233.68.129/10.233.68.129:36634] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5293d98d; decorates RemoteLauncher[hudson.remoting.Channel@5c716a0e:JNLP4-connect connection from 10.233.84.106/10.233.84.106:33394] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@381545d1; decorates RemoteLauncher[hudson.remoting.Channel@5ee3266f:JNLP4-connect connection from 10.233.68.156/10.233.68.156:55926] will be ignored (a typical symptom is the Git executable not being run inside a designated container) [Pipeline] { Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { [Pipeline] { [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6f28909e; decorates RemoteLauncher[hudson.remoting.Channel@2873620d:JNLP4-connect connection from 10.233.93.74/10.233.93.74:59322] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3a36172; decorates RemoteLauncher[hudson.remoting.Channel@44256614:JNLP4-connect connection from 10.233.100.84/10.233.100.84:60854] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] cache start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:24:59 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/091fd5bf-984b-4bcd-8db0-c46f73c8078c {"id":"091fd5bf-984b-4bcd-8db0-c46f73c8078c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973096} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5c78a5c8 091fd5bf-984b-4bcd-8db0-c46f73c8078c /tidb/cdc/default/default/upstream/7365753301818499302 {"id":7365753301818499302,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/091fd5bf-984b-4bcd-8db0-c46f73c8078c {"id":"091fd5bf-984b-4bcd-8db0-c46f73c8078c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973096} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5c78a5c8 091fd5bf-984b-4bcd-8db0-c46f73c8078c /tidb/cdc/default/default/upstream/7365753301818499302 {"id":7365753301818499302,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/091fd5bf-984b-4bcd-8db0-c46f73c8078c {"id":"091fd5bf-984b-4bcd-8db0-c46f73c8078c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973096} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5c78a5c8 091fd5bf-984b-4bcd-8db0-c46f73c8078c /tidb/cdc/default/default/upstream/7365753301818499302 {"id":7365753301818499302,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 13:24:59 CST 2024] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.26432645.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdcdown.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_datadown --cluster-id default --addr 127.0.0.1:8400 --pd http://127.0.0.1:2479 ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8400; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8400 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8400 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:25:02 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a6c2652f-0863-488a-beb9-39ab029b9a67 {"id":"a6c2652f-0863-488a-beb9-39ab029b9a67","address":"127.0.0.1:8400","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973099} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118f4c5c78d889 a6c2652f-0863-488a-beb9-39ab029b9a67 /tidb/cdc/default/default/upstream/7365753299455028408 {"id":7365753299455028408,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a6c2652f-0863-488a-beb9-39ab029b9a67 {"id":"a6c2652f-0863-488a-beb9-39ab029b9a67","address":"127.0.0.1:8400","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973099} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118f4c5c78d889 a6c2652f-0863-488a-beb9-39ab029b9a67 /tidb/cdc/default/default/upstream/7365753299455028408 {"id":7365753299455028408,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a6c2652f-0863-488a-beb9-39ab029b9a67 {"id":"a6c2652f-0863-488a-beb9-39ab029b9a67","address":"127.0.0.1:8400","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973099} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118f4c5c78d889 a6c2652f-0863-488a-beb9-39ab029b9a67 /tidb/cdc/default/default/upstream/7365753299455028408 {"id":7365753299455028408,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.2703.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306 -c test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/bdr_mode/conf/up.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365753301818499302,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306","create_time":"2024-05-06T13:25:02.921396259+08:00","start_ts":449569909053063172,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"enable_table_monitor":false,"bdr_mode":true,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569909053063172,"checkpoint_ts":449569909053063172,"checkpoint_time":"2024-05-06 13:25:02.772"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.2738.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:4000 -c test-2 --server http://127.0.0.1:8400 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/bdr_mode/conf/down.toml Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Create changefeed successfully! ID: test-2 Info: {"upstream_id":7365753299455028408,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:4000","create_time":"2024-05-06T13:25:04.873425357+08:00","start_ts":449569909565292552,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":true,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569909565292552,"checkpoint_ts":449569909565292552,"checkpoint_time":"2024-05-06 13:25:04.726"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3173b690001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5vlz1-83nq6-835n1, pid:1222, start at 2024-05-06 13:25:09.713932401 +0800 CST m=+5.248263755 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:27:09.720 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:25:09.719 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:15:09.719 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3173b690001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5vlz1-83nq6-835n1, pid:1222, start at 2024-05-06 13:25:09.713932401 +0800 CST m=+5.248263755 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:27:09.720 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:25:09.719 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:15:09.719 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3173b9ac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5vlz1-83nq6-835n1, pid:1285, start at 2024-05-06 13:25:09.902968179 +0800 CST m=+5.374844262 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:27:09.909 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:25:09.867 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:15:09.867 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 10.66 secs (349843763 bytes/sec) [Pipeline] { [Pipeline] cache [Mon May 6 13:25:12 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.25952597.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:25:15 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3934cada-527b-43ce-9032-3722aeda25cd {"id":"3934cada-527b-43ce-9032-3722aeda25cd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973113} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5cc564c0 3934cada-527b-43ce-9032-3722aeda25cd /tidb/cdc/default/default/upstream/7365753383248221995 {"id":7365753383248221995,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3934cada-527b-43ce-9032-3722aeda25cd {"id":"3934cada-527b-43ce-9032-3722aeda25cd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973113} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5cc564c0 3934cada-527b-43ce-9032-3722aeda25cd /tidb/cdc/default/default/upstream/7365753383248221995 {"id":7365753383248221995,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3934cada-527b-43ce-9032-3722aeda25cd {"id":"3934cada-527b-43ce-9032-3722aeda25cd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973113} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5cc564c0 3934cada-527b-43ce-9032-3722aeda25cd /tidb/cdc/default/default/upstream/7365753383248221995 {"id":7365753383248221995,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: a53a76f3-3f50-427e-a3d9-1e8e2e1f4809 Info: {"upstream_id":7365753383248221995,"namespace":"default","id":"a53a76f3-3f50-427e-a3d9-1e8e2e1f4809","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:25:16.156488887+08:00","start_ts":449569911686823937,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569911686823937,"checkpoint_ts":449569911686823937,"checkpoint_time":"2024-05-06 13:25:12.819"} Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 7.49 secs (498158609 bytes/sec) [Pipeline] { [Pipeline] cache table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 not exists for 1-th check, retry later table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 4-th check, retry later table test.finish_mark not exists for 5-th check, retry later table test.finish_mark not exists for 6-th check, retry later table test.finish_mark exists check table exists success check diff successfully Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 12.05 secs (309447415 bytes/sec) [Pipeline] { [Pipeline] cache wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:25:39 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> table tidb_cdc.syncpoint_v1 exists table tidb_cdc.syncpoint_v1 does not exists table bdr_mode.finish_mark exists check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:25:50 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 21.57 secs (172907035 bytes/sec) [Pipeline] { [Pipeline] cache start tidb cluster in /tmp/tidb_cdc_test/capture_suicide_while_balance_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3176f0ec0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5vlz1-83nq6-835n1, pid:3718, start at 2024-05-06 13:26:02.561839803 +0800 CST m=+5.365583663 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:28:02.568 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:26:02.555 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:16:02.555 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3176f0ec0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5vlz1-83nq6-835n1, pid:3718, start at 2024-05-06 13:26:02.561839803 +0800 CST m=+5.365583663 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:28:02.568 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:26:02.555 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:16:02.555 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3176f1b40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5vlz1-83nq6-835n1, pid:3790, start at 2024-05-06 13:26:02.638455165 +0800 CST m=+5.355061091 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:28:02.645 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:26:02.605 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:16:02.605 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Mon May 6 13:26:07 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.50365038.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:26:10 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe5f112e-3ba3-4330-9829-d2b30390f4da {"id":"fe5f112e-3ba3-4330-9829-d2b30390f4da","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973167} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5d9366ce fe5f112e-3ba3-4330-9829-d2b30390f4da /tidb/cdc/default/default/upstream/7365753611964539765 {"id":7365753611964539765,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe5f112e-3ba3-4330-9829-d2b30390f4da {"id":"fe5f112e-3ba3-4330-9829-d2b30390f4da","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973167} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5d9366ce fe5f112e-3ba3-4330-9829-d2b30390f4da /tidb/cdc/default/default/upstream/7365753611964539765 {"id":7365753611964539765,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe5f112e-3ba3-4330-9829-d2b30390f4da {"id":"fe5f112e-3ba3-4330-9829-d2b30390f4da","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973167} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5d9366ce fe5f112e-3ba3-4330-9829-d2b30390f4da /tidb/cdc/default/default/upstream/7365753611964539765 {"id":7365753611964539765,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.5084.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-manager Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7365753611964539765,"namespace":"default","id":"ddl-manager","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-06T13:26:11.019016865+08:00","start_ts":449569926913720325,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569926913720325,"checkpoint_ts":449569926913720325,"checkpoint_time":"2024-05-06 13:26:10.905"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31779ce00010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d, pid:3881, start at 2024-05-06 13:26:13.580773102 +0800 CST m=+5.382859915 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:28:13.591 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:26:13.560 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:16:13.560 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 10.26 secs (363412668 bytes/sec) [Pipeline] { [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31779ce00010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d, pid:3881, start at 2024-05-06 13:26:13.580773102 +0800 CST m=+5.382859915 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:28:13.591 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:26:13.560 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:16:13.560 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31779e680011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d, pid:3964, start at 2024-05-06 13:26:13.676795338 +0800 CST m=+5.408763933 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:28:13.686 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:26:13.658 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:16:13.658 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process 5041 exit for 1-th time... wait process 5041 exit for 2-th time... wait process 5041 exit for 3-th time... wait process 5041 exit for 4-th time... [Mon May 6 13:26:18 CST 2024] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.53235325.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5041) - No such process wait process 5041 exit for 5-th time... process 5041 already exit [Mon May 6 13:26:18 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.51605162.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:26:21 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e9c14d47-a53e-4b13-9fc6-96c1cf2b052d {"id":"e9c14d47-a53e-4b13-9fc6-96c1cf2b052d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973178} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5dbe62cb e9c14d47-a53e-4b13-9fc6-96c1cf2b052d /tidb/cdc/default/default/upstream/7365753660752549729 {"id":7365753660752549729,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e9c14d47-a53e-4b13-9fc6-96c1cf2b052d {"id":"e9c14d47-a53e-4b13-9fc6-96c1cf2b052d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973178} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5dbe62cb e9c14d47-a53e-4b13-9fc6-96c1cf2b052d /tidb/cdc/default/default/upstream/7365753660752549729 {"id":7365753660752549729,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e9c14d47-a53e-4b13-9fc6-96c1cf2b052d {"id":"e9c14d47-a53e-4b13-9fc6-96c1cf2b052d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973178} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5dbe62cb e9c14d47-a53e-4b13-9fc6-96c1cf2b052d /tidb/cdc/default/default/upstream/7365753660752549729 {"id":7365753660752549729,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 13:26:21 CST 2024] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.53775379.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:26:21 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365753611964539765 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 13:26:11.019016865 +0800 CST StartTs:449569926913720325 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025782d0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569926940196867} {CheckpointTs:449569927333412881 MinTableBarrierTs:449569927333412881 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/80f30b36-3276-4f31-8e76-e404cafa728d {"id":"80f30b36-3276-4f31-8e76-e404cafa728d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973178} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5d936952 80f30b36-3276-4f31-8e76-e404cafa728d /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365753611964539765,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T13:26:11.019016865+08:00","start-ts":449569926913720325,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569926940196867} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449569927346520113,"min-table-barrier-ts":449569927346520113,"admin-job-type":0} /tidb/cdc/default/default/task/position/80f30b36-3276-4f31-8e76-e404cafa728d/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365753611964539765 {"id":7365753611964539765,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365753611964539765 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 13:26:11.019016865 +0800 CST StartTs:449569926913720325 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025782d0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569926940196867} {CheckpointTs:449569927333412881 MinTableBarrierTs:449569927333412881 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/80f30b36-3276-4f31-8e76-e404cafa728d {"id":"80f30b36-3276-4f31-8e76-e404cafa728d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973178} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5d936952 80f30b36-3276-4f31-8e76-e404cafa728d /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365753611964539765,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T13:26:11.019016865+08:00","start-ts":449569926913720325,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569926940196867} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449569927346520113,"min-table-barrier-ts":449569927346520113,"admin-job-type":0} /tidb/cdc/default/default/task/position/80f30b36-3276-4f31-8e76-e404cafa728d/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365753611964539765 {"id":7365753611964539765,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365753611964539765 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 13:26:11.019016865 +0800 CST StartTs:449569926913720325 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025782d0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569926940196867} {CheckpointTs:449569927333412881 MinTableBarrierTs:449569927333412881 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/80f30b36-3276-4f31-8e76-e404cafa728d {"id":"80f30b36-3276-4f31-8e76-e404cafa728d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973178} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5d936952 80f30b36-3276-4f31-8e76-e404cafa728d /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365753611964539765,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T13:26:11.019016865+08:00","start-ts":449569926913720325,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569926940196867} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449569927346520113,"min-table-barrier-ts":449569927346520113,"admin-job-type":0} /tidb/cdc/default/default/task/position/80f30b36-3276-4f31-8e76-e404cafa728d/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365753611964539765 {"id":7365753611964539765,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 13:26:21 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.52155217.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:26:23 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365753611964539765 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 13:26:11.019016865 +0800 CST StartTs:449569926913720325 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025782d0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569926940196867} {CheckpointTs:449569927346520113 MinTableBarrierTs:449569927346520113 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/80f30b36-3276-4f31-8e76-e404cafa728d {"id":"80f30b36-3276-4f31-8e76-e404cafa728d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973178} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5d936952 80f30b36-3276-4f31-8e76-e404cafa728d /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365753611964539765,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T13:26:11.019016865+08:00","start-ts":449569926913720325,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569926940196867} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449569927346520113,"min-table-barrier-ts":449569927346520113,"admin-job-type":0} /tidb/cdc/default/default/task/position/80f30b36-3276-4f31-8e76-e404cafa728d/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365753611964539765 {"id":7365753611964539765,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365753611964539765 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 13:26:11.019016865 +0800 CST StartTs:449569926913720325 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025782d0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569926940196867} {CheckpointTs:449569927346520113 MinTableBarrierTs:449569927346520113 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/80f30b36-3276-4f31-8e76-e404cafa728d {"id":"80f30b36-3276-4f31-8e76-e404cafa728d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973178} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5d936952 80f30b36-3276-4f31-8e76-e404cafa728d /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365753611964539765,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T13:26:11.019016865+08:00","start-ts":449569926913720325,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569926940196867} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449569927346520113,"min-table-barrier-ts":449569927346520113,"admin-job-type":0} /tidb/cdc/default/default/task/position/80f30b36-3276-4f31-8e76-e404cafa728d/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365753611964539765 {"id":7365753611964539765,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365753611964539765 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 13:26:11.019016865 +0800 CST StartTs:449569926913720325 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025782d0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569926940196867} {CheckpointTs:449569927346520113 MinTableBarrierTs:449569927346520113 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/80f30b36-3276-4f31-8e76-e404cafa728d {"id":"80f30b36-3276-4f31-8e76-e404cafa728d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973178} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5d936952 80f30b36-3276-4f31-8e76-e404cafa728d /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365753611964539765,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T13:26:11.019016865+08:00","start-ts":449569926913720325,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569926940196867} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449569927346520113,"min-table-barrier-ts":449569927346520113,"admin-job-type":0} /tidb/cdc/default/default/task/position/80f30b36-3276-4f31-8e76-e404cafa728d/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365753611964539765 {"id":7365753611964539765,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 6.71 secs (555644606 bytes/sec) [Pipeline] { [Pipeline] cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:26:24 GMT < Content-Length: 1259 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09cc1fca-b611-47a6-95e7-a34900166858 {"id":"09cc1fca-b611-47a6-95e7-a34900166858","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973182} /tidb/cdc/default/__cdc_meta__/capture/e9c14d47-a53e-4b13-9fc6-96c1cf2b052d {"id":"e9c14d47-a53e-4b13-9fc6-96c1cf2b052d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973178} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5dbe62cb e9c14d47-a53e-4b13-9fc6-96c1cf2b052d /tidb/cdc/default/__cdc_meta__/owner/22318f4c5dbe6301 09cc1fca-b611-47a6-95e7-a34900166858 /tidb/cdc/default/default/upstream/7365753660752549729 {"id":7365753660752549729,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09cc1fca-b611-47a6-95e7-a34900166858 {"id":"09cc1fca-b611-47a6-95e7-a34900166858","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973182} /tidb/cdc/default/__cdc_meta__/capture/e9c14d47-a53e-4b13-9fc6-96c1cf2b052d {"id":"e9c14d47-a53e-4b13-9fc6-96c1cf2b052d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973178} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5dbe62cb e9c14d47-a53e-4b13-9fc6-96c1cf2b052d /tidb/cdc/default/__cdc_meta__/owner/22318f4c5dbe6301 09cc1fca-b611-47a6-95e7-a34900166858 /tidb/cdc/default/default/upstream/7365753660752549729 {"id":7365753660752549729,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09cc1fca-b611-47a6-95e7-a34900166858 {"id":"09cc1fca-b611-47a6-95e7-a34900166858","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973182} /tidb/cdc/default/__cdc_meta__/capture/e9c14d47-a53e-4b13-9fc6-96c1cf2b052d {"id":"e9c14d47-a53e-4b13-9fc6-96c1cf2b052d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973178} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5dbe62cb e9c14d47-a53e-4b13-9fc6-96c1cf2b052d /tidb/cdc/default/__cdc_meta__/owner/22318f4c5dbe6301 09cc1fca-b611-47a6-95e7-a34900166858 /tidb/cdc/default/default/upstream/7365753660752549729 {"id":7365753660752549729,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 2-th check, retry later table capture_suicide_while_balance_table.t1 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 3-th check, retry later table capture_suicide_while_balance_table.t1 exists table capture_suicide_while_balance_table.t2 not exists for 1-th check, retry later table capture_suicide_while_balance_table.t2 exists table capture_suicide_while_balance_table.t3 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 4-th check, retry later table ddl_manager.finish_mark not exists for 5-th check, retry later table capture_suicide_while_balance_table.t3 exists table capture_suicide_while_balance_table.t4 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 6-th check, retry later table capture_suicide_while_balance_table.t4 exists Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 7.58 secs (492188744 bytes/sec) [Pipeline] { [Pipeline] cache table ddl_manager.finish_mark not exists for 7-th check, retry later % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 140 100 35 100 105 423 1269 --:--:-- --:--:-- --:--:-- 1280 { "status": true, "message": "" table ddl_manager.finish_mark not exists for 8-th check, retry later }lease 22318f4c5dbe6301 revoked check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 9-th check, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:26:40 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> table ddl_manager.finish_mark not exists for 10-th check, retry later table ddl_manager.finish_mark not exists for 11-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 6.77 secs (551349357 bytes/sec) [Pipeline] { [Pipeline] cache table ddl_manager.finish_mark not exists for 12-th check, retry later table ddl_manager.finish_mark not exists for 13-th check, retry later table ddl_manager.finish_mark not exists for 14-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 15-th check, retry later table ddl_manager.finish_mark not exists for 16-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 7.53 secs (495123093 bytes/sec) [Pipeline] { [Pipeline] cache start tidb cluster in /tmp/tidb_cdc_test/syncpoint Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 17-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 18-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 19-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 20-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 21-th check, retry later table ddl_manager.finish_mark not exists for 22-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317aaf080012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d, pid:6689, start at 2024-05-06 13:27:03.901946217 +0800 CST m=+6.478631559 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:03.911 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:03.874 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:03.874 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317aaf080012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d, pid:6689, start at 2024-05-06 13:27:03.901946217 +0800 CST m=+6.478631559 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:03.911 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:03.874 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:03.874 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317ab07c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d, pid:6758, start at 2024-05-06 13:27:04.000443449 +0800 CST m=+6.509396884 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:04.011 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:04.016 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:04.016 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/syncpoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/syncpoint/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 23-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.8136.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 24-th check, retry later + set +x + tso='449569942083731457 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449569942083731457 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:27:10 CST 2024] <<<<<< START cdc server in syncpoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/DDLSinkInitializeSlowly=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.81738175.out server --log-file /tmp/tidb_cdc_test/syncpoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/syncpoint/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 25-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:27:13 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97b8f0e8-d71a-4b0d-b835-0c5c4ea6b9c4 {"id":"97b8f0e8-d71a-4b0d-b835-0c5c4ea6b9c4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973230} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5e847be1 97b8f0e8-d71a-4b0d-b835-0c5c4ea6b9c4 /tidb/cdc/default/default/upstream/7365753870256993367 {"id":7365753870256993367,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97b8f0e8-d71a-4b0d-b835-0c5c4ea6b9c4 {"id":"97b8f0e8-d71a-4b0d-b835-0c5c4ea6b9c4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973230} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5e847be1 97b8f0e8-d71a-4b0d-b835-0c5c4ea6b9c4 /tidb/cdc/default/default/upstream/7365753870256993367 {"id":7365753870256993367,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97b8f0e8-d71a-4b0d-b835-0c5c4ea6b9c4 {"id":"97b8f0e8-d71a-4b0d-b835-0c5c4ea6b9c4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973230} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5e847be1 97b8f0e8-d71a-4b0d-b835-0c5c4ea6b9c4 /tidb/cdc/default/default/upstream/7365753870256993367 {"id":7365753870256993367,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.8235.out cli changefeed create --start-ts=449569942083731457 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/syncpoint/conf/changefeed.toml table ddl_manager.finish_mark not exists for 26-th check, retry later Create changefeed successfully! ID: 05a08c53-2759-4c04-a441-2a7420a096a1 Info: {"upstream_id":7365753870256993367,"namespace":"default","id":"05a08c53-2759-4c04-a441-2a7420a096a1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T13:27:13.785620996+08:00","start_ts":449569942083731457,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569942083731457,"checkpoint_ts":449569942083731457,"checkpoint_time":"2024-05-06 13:27:08.774"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table ddl_manager.finish_mark not exists for 27-th check, retry later ***************** properties ***************** "mysql.db"="testSync" "insertproportion"="0" "workload"="core" "recordcount"="10" "operationcount"="0" "mysql.port"="4000" "scanproportion"="0" "dotransactions"="false" "threadcount"="2" "mysql.user"="root" "readallfields"="true" "readproportion"="0" "updateproportion"="0" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 9.59488ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1943.9, Avg(us): 1808, Min(us): 1092, Max(us): 4329, 95th(us): 5000, 99th(us): 5000 table ddl_manager.finish_mark not exists for 28-th check, retry later table ddl_manager.finish_mark not exists for 29-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 25.22 secs (147919089 bytes/sec) [Pipeline] { [Pipeline] cache table ddl_manager.finish_mark not exists for 30-th check, retry later table ddl_manager.finish_mark not exists for 31-th check, retry later table ddl_manager.finish_mark not exists for 32-th check, retry later table ddl_manager.finish_mark not exists for 33-th check, retry later table ddl_manager.finish_mark not exists for 34-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 7.56 secs (493705824 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=8ac88e87-6b94-42b4-bb0f-536701579a43 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-60ww2 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1779-60ww2 pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G01 Run cases: http_api http_api_tls api_v2 http_api_tls_with_user_auth cli_tls_with_auth kafka_simple_basic kafka_simple_basic_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_simple_claim_check kafka_simple_claim_check_avro canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible multi_topics avro_basic canal_json_handle_key_only open_protocol_handle_key_only canal_json_claim_check open_protocol_claim_check canal_json_storage_basic canal_json_storage_partition_table multi_tables_ddl PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=92b27cc1-ba28-4181-8f4b-2e3eed787d93 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-mlls2 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1779-mlls2 pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=809ed446-9947-4351-9117-ae32c4d6afca BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-xwksv-n7b01-mfk0q HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-xwksv GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1779-xwksv pingcap-tiflow-pull-cdc-integration-test-1779-xwksv-n7b01-mfk0q GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-xwksv-n7b01-mfk0q GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh table ddl_manager.finish_mark not exists for 35-th check, retry later WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Collecting requests==2.26.0 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G08 Run cases: processor_err_chan changefeed_reconstruct multi_capture synced_status_with_redo PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=fdd0bf87-b7e9-4477-ac45-c8044603c17b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-rqdbn GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57 pingcap_tiflow_pull_cdc_integration_test_1779-rqdbn GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh [Pipeline] sh Downloading https://files.pythonhosted.org/packages/92/96/144f70b972a9c0eabbd4391ef93ccd49d0f2747f4f6a2a2738e99e5adc65/requests-2.26.0-py2.py3-none-any.whl (62kB) + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=ebb3543c-f4ec-4284-9c9b-4119fd729695 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-m22f9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd pingcap_tiflow_pull_cdc_integration_test_1779-m22f9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G10 Run cases: default_value simple cdc_server_tips event_filter sql_mode PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=350597cd-62ed-4350-bd45-b4c9b0e70e49 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-1lcp8 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn pingcap_tiflow_pull_cdc_integration_test_1779-1lcp8 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh Collecting certifi>=2017.4.17 (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/ba/06/a07f096c664aeb9f01624f858c3add0a4e913d6c96257acb4fce61e7de14/certifi-2024.2.2-py3-none-any.whl (163kB) + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=305bb2ed-d186-4974-a0d2-e2a6e9f8fcd7 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-2dpm3 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1779-2dpm3 pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh Collecting idna<4,>=2.5; python_version >= "3" (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/e5/3e/741d8c82801c347547f8a2a06aa57dbb1992be9e948df2ea0eda2c8b79e8/idna-3.7-py3-none-any.whl (66kB) + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G04 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G09 Run cases: gc_safepoint changefeed_pause_resume cli_with_auth savepoint synced_status PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=f2ee3424-0fcd-4c0b-96bb-bbefe7785b4d BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-kkw9l GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1779-kkw9l pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=71277ae7-55a2-408d-99a1-8db990818066 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-36hxm GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54 pingcap_tiflow_pull_cdc_integration_test_1779-36hxm GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 36-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G03 Run cases: row_format drop_many_tables processor_stop_delay partition_table PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=3bc955d3-0fb9-44dd-b0e9-3e7d934929f9 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-c7kfw GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3 pingcap_tiflow_pull_cdc_integration_test_1779-c7kfw GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G12 Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=7ffab128-2ef8-4681-9b16-88fc17559196 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-5155r-hgjhp-wsj0l HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-5155r GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1779-5155r pingcap-tiflow-pull-cdc-integration-test-1779-5155r-hgjhp-wsj0l GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-5155r-hgjhp-wsj0l GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: mysql... <<================= start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Collecting urllib3<1.27,>=1.21.1 (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/b0/53/aa91e163dcfd1e5b82d8a890ecf13314e3e149c05270cc644581f77f17fd/urllib3-1.26.18-py2.py3-none-any.whl (143kB) Installing collected packages: certifi, idna, urllib3, requests Found existing installation: certifi 2023.5.7 Uninstalling certifi-2023.5.7: Successfully uninstalled certifi-2023.5.7 Found existing installation: idna 3.4 Uninstalling idna-3.4: Successfully uninstalled idna-3.4 Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Found existing installation: urllib3 1.26.16 Uninstalling urllib3-1.26.16: Successfully uninstalled urllib3-1.26.16 Found existing installation: requests 2.27.1 Uninstalling requests-2.27.1: Successfully uninstalled requests-2.27.1 Successfully installed certifi-2024.2.2 idna-3.7 requests-2.26.0 urllib3-1.26.18 start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 37-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... The 1 times to try to start tidb cluster... The 1 times to try to start tidb cluster... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_ddl Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 38-th check, retry later The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] container [Pipeline] { Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/http_api Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiDB... table ddl_manager.finish_mark not exists for 39-th check, retry later Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min table ddl_manager.finish_mark not exists for 40-th check, retry later [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] { ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] withCredentials Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { table ddl_manager.finish_mark not exists for 41-th check, retry later [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d28d80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv, pid:1239, start at 2024-05-06 13:27:44.475502173 +0800 CST m=+5.249897151 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:44.483 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:44.487 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:44.487 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 42-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d3fe00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57, pid:1226, start at 2024-05-06 13:27:45.958182133 +0800 CST m=+5.743672681 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:45.970 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:45.962 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:45.962 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d28d80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv, pid:1239, start at 2024-05-06 13:27:44.475502173 +0800 CST m=+5.249897151 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:44.483 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:44.487 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:44.487 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d2b380014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv, pid:1320, start at 2024-05-06 13:27:44.621835556 +0800 CST m=+5.341949692 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:44.627 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:44.590 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:44.590 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 43-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d3fe00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57, pid:1226, start at 2024-05-06 13:27:45.958182133 +0800 CST m=+5.743672681 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:45.970 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:45.962 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:45.962 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d416c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57, pid:1297, start at 2024-05-06 13:27:46.058230499 +0800 CST m=+5.772408248 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:46.066 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:46.062 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:46.062 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d45b00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-xwksv-n7b01-mfk0q, pid:1242, start at 2024-05-06 13:27:46.324556455 +0800 CST m=+5.545438461 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:46.333 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:46.334 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:46.334 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d45b00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-xwksv-n7b01-mfk0q, pid:1242, start at 2024-05-06 13:27:46.324556455 +0800 CST m=+5.545438461 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:46.333 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:46.334 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:46.334 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d47300014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-xwksv-n7b01-mfk0q, pid:1323, start at 2024-05-06 13:27:46.411664956 +0800 CST m=+5.578321720 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:46.420 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:46.380 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:46.380 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d67ec0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3, pid:1166, start at 2024-05-06 13:27:48.480335467 +0800 CST m=+5.628624734 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:48.488 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:48.475 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:48.475 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2645.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d65180012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd, pid:1174, start at 2024-05-06 13:27:48.322101133 +0800 CST m=+5.323222177 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:48.331 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:48.294 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:48.294 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d65180012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd, pid:1174, start at 2024-05-06 13:27:48.322101133 +0800 CST m=+5.323222177 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:48.331 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:48.294 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:48.294 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d67600014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd, pid:1261, start at 2024-05-06 13:27:48.464412812 +0800 CST m=+5.397643289 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:48.472 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:48.440 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:48.440 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 44-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d5ec00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz, pid:1233, start at 2024-05-06 13:27:47.893067382 +0800 CST m=+5.499765699 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:47.900 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:47.888 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:47.888 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d5ec00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz, pid:1233, start at 2024-05-06 13:27:47.893067382 +0800 CST m=+5.499765699 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:47.900 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:47.888 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:47.888 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d5d140016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz, pid:1320, start at 2024-05-06 13:27:47.816102644 +0800 CST m=+5.365882878 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:47.823 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:47.781 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:47.781 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d7bd40008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54, pid:1170, start at 2024-05-06 13:27:49.757146002 +0800 CST m=+5.249175443 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:49.764 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:49.749 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:49.749 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d8334000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:1366, start at 2024-05-06 13:27:50.232712089 +0800 CST m=+5.240943503 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:50.240 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:50.221 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:50.221 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d8334000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:1366, start at 2024-05-06 13:27:50.232712089 +0800 CST m=+5.240943503 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:50.240 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:50.221 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:50.221 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d831c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:1450, start at 2024-05-06 13:27:50.255975586 +0800 CST m=+5.209997897 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:50.263 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:50.265 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:50.265 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d67ec0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3, pid:1166, start at 2024-05-06 13:27:48.480335467 +0800 CST m=+5.628624734 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:48.488 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:48.475 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:48.475 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d695c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3, pid:1240, start at 2024-05-06 13:27:48.56858154 +0800 CST m=+5.637539731 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:48.575 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:48.567 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:48.567 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/default_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/default_value/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/default_value/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='449569952861519873 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449569952861519873 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:27:51 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.26312633.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ***************** properties ***************** "mysql.host"="127.0.0.1" "scanproportion"="0" "threadcount"="2" "requestdistribution"="uniform" "workload"="core" "dotransactions"="false" "readproportion"="0" "recordcount"="10" "mysql.db"="sink_retry" "readallfields"="true" "operationcount"="0" "mysql.port"="4000" "updateproportion"="0" "mysql.user"="root" "insertproportion"="0" ********************************************** Run finished, takes 9.564827ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1840.5, Avg(us): 1767, Min(us): 977, Max(us): 4062, 95th(us): 5000, 99th(us): 5000 [Mon May 6 13:27:51 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.27052707.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 13:27:51 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.26332635.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 45-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d70d40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d, pid:1384, start at 2024-05-06 13:27:49.083155176 +0800 CST m=+5.469047590 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:49.090 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:49.095 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:49.095 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d70d40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d, pid:1384, start at 2024-05-06 13:27:49.083155176 +0800 CST m=+5.469047590 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:49.090 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:49.095 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:49.095 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d70c00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d, pid:1458, start at 2024-05-06 13:27:49.064935017 +0800 CST m=+5.383780872 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:49.072 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:49.040 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:49.040 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/http_api/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_api/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/http_api/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 13:27:52 CST 2024] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.26322634.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d7bd40008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54, pid:1170, start at 2024-05-06 13:27:49.757146002 +0800 CST m=+5.249175443 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:49.764 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:49.749 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:49.749 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317d7e280006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54, pid:1251, start at 2024-05-06 13:27:49.90330419 +0800 CST m=+5.327410657 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:49.909 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:49.898 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:49.898 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2693.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2618.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317da7940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd, pid:1362, start at 2024-05-06 13:27:52.58855694 +0800 CST m=+5.348804299 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:52.596 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:52.598 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:52.598 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 13:27:52 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/ProcessorAddTableError=1*return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.27442746.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 46-th check, retry later [Mon May 6 13:27:53 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.27112713.out server --log-file /tmp/tidb_cdc_test/http_api/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:27:54 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e2b8a9bd-0f1d-4ce8-88f0-19114b8f8445 {"id":"e2b8a9bd-0f1d-4ce8-88f0-19114b8f8445","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973271} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f260ccb e2b8a9bd-0f1d-4ce8-88f0-19114b8f8445 /tidb/cdc/default/default/upstream/7365754054126571806 {"id":7365754054126571806,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e2b8a9bd-0f1d-4ce8-88f0-19114b8f8445 {"id":"e2b8a9bd-0f1d-4ce8-88f0-19114b8f8445","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973271} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f260ccb e2b8a9bd-0f1d-4ce8-88f0-19114b8f8445 /tidb/cdc/default/default/upstream/7365754054126571806 {"id":7365754054126571806,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e2b8a9bd-0f1d-4ce8-88f0-19114b8f8445 {"id":"e2b8a9bd-0f1d-4ce8-88f0-19114b8f8445","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973271} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f260ccb e2b8a9bd-0f1d-4ce8-88f0-19114b8f8445 /tidb/cdc/default/default/upstream/7365754054126571806 {"id":7365754054126571806,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:27:54 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/be9d05e5-ec7c-45c8-8899-ec23d7402bee {"id":"be9d05e5-ec7c-45c8-8899-ec23d7402bee","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973271} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f3296c3 be9d05e5-ec7c-45c8-8899-ec23d7402bee /tidb/cdc/default/default/upstream/7365754063503306118 {"id":7365754063503306118,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/be9d05e5-ec7c-45c8-8899-ec23d7402bee {"id":"be9d05e5-ec7c-45c8-8899-ec23d7402bee","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973271} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f3296c3 be9d05e5-ec7c-45c8-8899-ec23d7402bee /tidb/cdc/default/default/upstream/7365754063503306118 {"id":7365754063503306118,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/be9d05e5-ec7c-45c8-8899-ec23d7402bee {"id":"be9d05e5-ec7c-45c8-8899-ec23d7402bee","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973271} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f3296c3 be9d05e5-ec7c-45c8-8899-ec23d7402bee /tidb/cdc/default/default/upstream/7365754063503306118 {"id":7365754063503306118,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2622.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:27:54 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/61bf49ed-04d0-4f84-aa78-c62e8dde5aa6 {"id":"61bf49ed-04d0-4f84-aa78-c62e8dde5aa6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973272} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f23e9ee 61bf49ed-04d0-4f84-aa78-c62e8dde5aa6 /tidb/cdc/default/default/upstream/7365754048258085853 {"id":7365754048258085853,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/61bf49ed-04d0-4f84-aa78-c62e8dde5aa6 {"id":"61bf49ed-04d0-4f84-aa78-c62e8dde5aa6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973272} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f23e9ee 61bf49ed-04d0-4f84-aa78-c62e8dde5aa6 /tidb/cdc/default/default/upstream/7365754048258085853 {"id":7365754048258085853,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/61bf49ed-04d0-4f84-aa78-c62e8dde5aa6 {"id":"61bf49ed-04d0-4f84-aa78-c62e8dde5aa6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973272} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f23e9ee 61bf49ed-04d0-4f84-aa78-c62e8dde5aa6 /tidb/cdc/default/default/upstream/7365754048258085853 {"id":7365754048258085853,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2768.out cli changefeed create --start-ts=449569952861519873 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' 0 + set +x + tso='449569953840365569 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449569953840365569 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:27:55 CST 2024] <<<<<< START cdc server in default_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.27372739.out server --log-file /tmp/tidb_cdc_test/default_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/default_value/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: fdf47160-3969-424c-a953-d99425afe425 Info: {"upstream_id":7365754048258085853,"namespace":"default","id":"fdf47160-3969-424c-a953-d99425afe425","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T13:27:55.232173755+08:00","start_ts":449569952861519873,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569952861519873,"checkpoint_ts":449569952861519873,"checkpoint_time":"2024-05-06 13:27:49.888"} PASS check diff failed 1-th time, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + tso='449569953854259201 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449569953854259201 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:27:55 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.26542656.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317da9d40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5155r-hgjhp-wsj0l, pid:1297, start at 2024-05-06 13:27:52.727519221 +0800 CST m=+5.484067122 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:52.737 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:52.743 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:52.743 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317da9d40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5155r-hgjhp-wsj0l, pid:1297, start at 2024-05-06 13:27:52.727519221 +0800 CST m=+5.484067122 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:52.737 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:52.743 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:52.743 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317da9c4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5155r-hgjhp-wsj0l, pid:1379, start at 2024-05-06 13:27:52.704437309 +0800 CST m=+5.373279590 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:52.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:52.689 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:52.689 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 47-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:27:55 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f6527aff-2272-4a26-920b-2358d79303cf {"id":"f6527aff-2272-4a26-920b-2358d79303cf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973273} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f2c4ec9 f6527aff-2272-4a26-920b-2358d79303cf /tidb/cdc/default/default/upstream/7365754062463967314 {"id":7365754062463967314,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f6527aff-2272-4a26-920b-2358d79303cf {"id":"f6527aff-2272-4a26-920b-2358d79303cf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973273} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f2c4ec9 f6527aff-2272-4a26-920b-2358d79303cf /tidb/cdc/default/default/upstream/7365754062463967314 {"id":7365754062463967314,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f6527aff-2272-4a26-920b-2358d79303cf {"id":"f6527aff-2272-4a26-920b-2358d79303cf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973273} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f2c4ec9 f6527aff-2272-4a26-920b-2358d79303cf /tidb/cdc/default/default/upstream/7365754062463967314 {"id":7365754062463967314,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317da7940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd, pid:1362, start at 2024-05-06 13:27:52.58855694 +0800 CST m=+5.348804299 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:52.596 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:52.598 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:52.598 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317da9240012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd, pid:1441, start at 2024-05-06 13:27:52.665793305 +0800 CST m=+5.343478318 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:29:52.674 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:27:52.649 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:17:52.649 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:27:55 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e04d34c8-bfb3-4102-bd8a-6321013e00f9 {"id":"e04d34c8-bfb3-4102-bd8a-6321013e00f9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973273} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f27b767 e04d34c8-bfb3-4102-bd8a-6321013e00f9 /tidb/cdc/default/default/upstream/7365754055922745422 {"id":7365754055922745422,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e04d34c8-bfb3-4102-bd8a-6321013e00f9 {"id":"e04d34c8-bfb3-4102-bd8a-6321013e00f9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973273} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f27b767 e04d34c8-bfb3-4102-bd8a-6321013e00f9 /tidb/cdc/default/default/upstream/7365754055922745422 {"id":7365754055922745422,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e04d34c8-bfb3-4102-bd8a-6321013e00f9 {"id":"e04d34c8-bfb3-4102-bd8a-6321013e00f9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973273} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f27b767 e04d34c8-bfb3-4102-bd8a-6321013e00f9 /tidb/cdc/default/default/upstream/7365754055922745422 {"id":7365754055922745422,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 5a6fbb73-465e-4836-8485-349174f23391 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=5a6fbb73-465e-4836-8485-349174f23391 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 5a6fbb73-465e-4836-8485-349174f23391 -s + info='{ "upstream_id": 7365754055922745422, "namespace": "default", "id": "5a6fbb73-465e-4836-8485-349174f23391", "state": "normal", "checkpoint_tso": 449569954480259075, "checkpoint_time": "2024-05-06 13:27:56.063", "error": null }' + echo '{ "upstream_id": 7365754055922745422, "namespace": "default", "id": "5a6fbb73-465e-4836-8485-349174f23391", "state": "normal", "checkpoint_tso": 449569954480259075, "checkpoint_time": "2024-05-06 13:27:56.063", "error": null }' { "upstream_id": 7365754055922745422, "namespace": "default", "id": "5a6fbb73-465e-4836-8485-349174f23391", "state": "normal", "checkpoint_tso": 449569954480259075, "checkpoint_time": "2024-05-06 13:27:56.063", "error": null } ++ echo '{' '"upstream_id":' 7365754055922745422, '"namespace":' '"default",' '"id":' '"5a6fbb73-465e-4836-8485-349174f23391",' '"state":' '"normal",' '"checkpoint_tso":' 449569954480259075, '"checkpoint_time":' '"2024-05-06' '13:27:56.063",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365754055922745422, '"namespace":' '"default",' '"id":' '"5a6fbb73-465e-4836-8485-349174f23391",' '"state":' '"normal",' '"checkpoint_tso":' 449569954480259075, '"checkpoint_time":' '"2024-05-06' '13:27:56.063",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + set +x + tso='449569954240659457 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449569954240659457 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:27:56 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.26592661.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x check diff failed 1-th time, retry later ***************** properties ***************** "workload"="core" "mysql.db"="consistent_replicate_ddl" "mysql.user"="root" "scanproportion"="0" "updateproportion"="0" "recordcount"="50" "operationcount"="0" "mysql.host"="127.0.0.1" "threadcount"="10" "mysql.port"="4000" "fieldcount"="3" "readproportion"="0" "dotransactions"="false" "requestdistribution"="uniform" "insertproportion"="0" "readallfields"="true" ********************************************** Run finished, takes 11.920399ms INSERT - Takes(s): 0.0, Count: 50, OPS: 6925.9, Avg(us): 2128, Min(us): 1045, Max(us): 5511, 95th(us): 6000, 99th(us): 6000 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:27:57 GMT < Content-Length: 853 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ea154161-b8c5-4c73-9e61-dae0975f7c79 {"id":"ea154161-b8c5-4c73-9e61-dae0975f7c79","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973274} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f38d431 ea154161-b8c5-4c73-9e61-dae0975f7c79 /tidb/cdc/default/default/upstream/7365754067370870491 {"id":7365754067370870491,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ea154161-b8c5-4c73-9e61-dae0975f7c79 {"id":"ea154161-b8c5-4c73-9e61-dae0975f7c79","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973274} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f38d431 ea154161-b8c5-4c73-9e61-dae0975f7c79 /tidb/cdc/default/default/upstream/7365754067370870491 {"id":7365754067370870491,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ea154161-b8c5-4c73-9e61-dae0975f7c79 {"id":"ea154161-b8c5-4c73-9e61-dae0975f7c79","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973274} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f38d431 ea154161-b8c5-4c73-9e61-dae0975f7c79 /tidb/cdc/default/default/upstream/7365754067370870491 {"id":7365754067370870491,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2716.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 2-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2703.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 48-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:27:58 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd6cb816-6710-4065-bbc5-33a85675051c {"id":"dd6cb816-6710-4065-bbc5-33a85675051c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973275} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f3666d5 dd6cb816-6710-4065-bbc5-33a85675051c /tidb/cdc/default/default/upstream/7365754069396725057 {"id":7365754069396725057,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd6cb816-6710-4065-bbc5-33a85675051c {"id":"dd6cb816-6710-4065-bbc5-33a85675051c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973275} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f3666d5 dd6cb816-6710-4065-bbc5-33a85675051c /tidb/cdc/default/default/upstream/7365754069396725057 {"id":7365754069396725057,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd6cb816-6710-4065-bbc5-33a85675051c {"id":"dd6cb816-6710-4065-bbc5-33a85675051c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973275} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f3666d5 dd6cb816-6710-4065-bbc5-33a85675051c /tidb/cdc/default/default/upstream/7365754069396725057 {"id":7365754069396725057,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2793.out cli changefeed create --start-ts=449569953840365569 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 5fb2f1af-3e51-4401-be19-9846de580506 Info: {"upstream_id":7365754069396725057,"namespace":"default","id":"5fb2f1af-3e51-4401-be19-9846de580506","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:27:58.612678009+08:00","start_ts":449569953840365569,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569953840365569,"checkpoint_ts":449569953840365569,"checkpoint_time":"2024-05-06 13:27:53.622"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... "is-owner": true, run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:27:58 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a402f064-8fd2-41f3-938e-8baba84b75a5 {"id":"a402f064-8fd2-41f3-938e-8baba84b75a5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973275} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f2ff8da a402f064-8fd2-41f3-938e-8baba84b75a5 /tidb/cdc/default/default/upstream/7365754058806119490 {"id":7365754058806119490,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a402f064-8fd2-41f3-938e-8baba84b75a5 {"id":"a402f064-8fd2-41f3-938e-8baba84b75a5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973275} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f2ff8da a402f064-8fd2-41f3-938e-8baba84b75a5 /tidb/cdc/default/default/upstream/7365754058806119490 {"id":7365754058806119490,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a402f064-8fd2-41f3-938e-8baba84b75a5 {"id":"a402f064-8fd2-41f3-938e-8baba84b75a5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973275} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f2ff8da a402f064-8fd2-41f3-938e-8baba84b75a5 /tidb/cdc/default/default/upstream/7365754058806119490 {"id":7365754058806119490,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2726.out cli changefeed create --start-ts=449569953854259201 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 228bcfb4-113d-4bd2-8474-ef72e63266cc Info: {"upstream_id":7365754058806119490,"namespace":"default","id":"228bcfb4-113d-4bd2-8474-ef72e63266cc","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:27:58.875598985+08:00","start_ts":449569953854259201,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569953854259201,"checkpoint_ts":449569953854259201,"checkpoint_time":"2024-05-06 13:27:53.675"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table consistent_replicate_ddl.usertable1 not exists for 1-th check, retry later check diff failed 2-th time, retry later + set +x + tso='449569954960244737 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449569954960244737 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:27:59 CST 2024] <<<<<< START cdc server in many_pk_or_uk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.27582760.out server --log-file /tmp/tidb_cdc_test/many_pk_or_uk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/many_pk_or_uk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully check_safepoint_forward http://127.0.0.1:2379 7365754063503306118 449569954999566337 449569954108276744 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:27:59 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e75bac77-20b1-4eaf-baac-23161a878dac {"id":"e75bac77-20b1-4eaf-baac-23161a878dac","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973276} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f329ad6 e75bac77-20b1-4eaf-baac-23161a878dac /tidb/cdc/default/default/upstream/7365754072969886062 {"id":7365754072969886062,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e75bac77-20b1-4eaf-baac-23161a878dac {"id":"e75bac77-20b1-4eaf-baac-23161a878dac","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973276} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f329ad6 e75bac77-20b1-4eaf-baac-23161a878dac /tidb/cdc/default/default/upstream/7365754072969886062 {"id":7365754072969886062,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e75bac77-20b1-4eaf-baac-23161a878dac {"id":"e75bac77-20b1-4eaf-baac-23161a878dac","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973276} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f329ad6 e75bac77-20b1-4eaf-baac-23161a878dac /tidb/cdc/default/default/upstream/7365754072969886062 {"id":7365754072969886062,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2720.out cli changefeed create --start-ts=449569954240659457 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ + set +x + set +x + tso='449569955118841857 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449569955118841857 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:27:59 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.27392741.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 49-th check, retry later Create changefeed successfully! ID: 3a7386a0-6ab0-4833-b89b-45d8341d5498 Info: {"upstream_id":7365754072969886062,"namespace":"default","id":"3a7386a0-6ab0-4833-b89b-45d8341d5498","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:28:00.142859396+08:00","start_ts":449569954240659457,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569954240659457,"checkpoint_ts":449569954240659457,"checkpoint_time":"2024-05-06 13:27:55.149"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... owner pid: 2716 owner id ea154161-b8c5-4c73-9e61-dae0975f7c79 default [Mon May 6 13:28:00 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.28482850.out server --log-file /tmp/tidb_cdc_test/http_api/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/google/uuid v1.6.0 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading golang.org/x/sync v0.7.0 go: downloading golang.org/x/time v0.5.0 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/multierr v1.11.0 + set +x run task successfully check_changefeed_state http://127.0.0.1:2379 4a92993c-a68c-4f38-92aa-f964dd456791 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=4a92993c-a68c-4f38-92aa-f964dd456791 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 4a92993c-a68c-4f38-92aa-f964dd456791 -s go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/sys v0.19.0 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda + info='{ "upstream_id": 7365754063503306118, "namespace": "default", "id": "4a92993c-a68c-4f38-92aa-f964dd456791", "state": "stopped", "checkpoint_tso": 449569955523854341, "checkpoint_time": "2024-05-06 13:28:00.044", "error": null }' + echo '{ "upstream_id": 7365754063503306118, "namespace": "default", "id": "4a92993c-a68c-4f38-92aa-f964dd456791", "state": "stopped", "checkpoint_tso": 449569955523854341, "checkpoint_time": "2024-05-06 13:28:00.044", "error": null }' { "upstream_id": 7365754063503306118, "namespace": "default", "id": "4a92993c-a68c-4f38-92aa-f964dd456791", "state": "stopped", "checkpoint_tso": 449569955523854341, "checkpoint_time": "2024-05-06 13:28:00.044", "error": null } ++ echo '{' '"upstream_id":' 7365754063503306118, '"namespace":' '"default",' '"id":' '"4a92993c-a68c-4f38-92aa-f964dd456791",' '"state":' '"stopped",' '"checkpoint_tso":' 449569955523854341, '"checkpoint_time":' '"2024-05-06' '13:28:00.044",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365754063503306118, '"namespace":' '"default",' '"id":' '"4a92993c-a68c-4f38-92aa-f964dd456791",' '"state":' '"stopped",' '"checkpoint_tso":' 449569955523854341, '"checkpoint_time":' '"2024-05-06' '13:28:00.044",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365754063503306118 go: downloading golang.org/x/text v0.14.0 check diff successfully table consistent_replicate_ddl.usertable1 exists table consistent_replicate_ddl.usertable2 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 50-th check, retry later + set +x wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:28:02 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/07313860-c318-4a7b-8e79-5b4ae5eae232 {"id":"07313860-c318-4a7b-8e79-5b4ae5eae232","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973279} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f3d51d6 07313860-c318-4a7b-8e79-5b4ae5eae232 /tidb/cdc/default/default/upstream/7365754085543089784 {"id":7365754085543089784,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/07313860-c318-4a7b-8e79-5b4ae5eae232 {"id":"07313860-c318-4a7b-8e79-5b4ae5eae232","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973279} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f3d51d6 07313860-c318-4a7b-8e79-5b4ae5eae232 /tidb/cdc/default/default/upstream/7365754085543089784 {"id":7365754085543089784,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/07313860-c318-4a7b-8e79-5b4ae5eae232 {"id":"07313860-c318-4a7b-8e79-5b4ae5eae232","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973279} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f3d51d6 07313860-c318-4a7b-8e79-5b4ae5eae232 /tidb/cdc/default/default/upstream/7365754085543089784 {"id":7365754085543089784,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2813.out cli changefeed create --start-ts=449569954960244737 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ wait process cdc.test exit for 3-th time... Create changefeed successfully! ID: e210cf48-3873-4491-8b66-e00abd75f6b4 Info: {"upstream_id":7365754085543089784,"namespace":"default","id":"e210cf48-3873-4491-8b66-e00abd75f6b4","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:28:02.885222499+08:00","start_ts":449569954960244737,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569954960244737,"checkpoint_ts":449569954960244737,"checkpoint_time":"2024-05-06 13:27:57.894"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/tidwall/btree v1.7.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:28:03 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/apache/thrift v0.16.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:28:03 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5b3cfc9d-9c68-4bf6-93e6-1e50e22753ea {"id":"5b3cfc9d-9c68-4bf6-93e6-1e50e22753ea","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973280} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f413fd6 5b3cfc9d-9c68-4bf6-93e6-1e50e22753ea /tidb/cdc/default/default/upstream/7365754086620651211 {"id":7365754086620651211,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5b3cfc9d-9c68-4bf6-93e6-1e50e22753ea {"id":"5b3cfc9d-9c68-4bf6-93e6-1e50e22753ea","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973280} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f413fd6 5b3cfc9d-9c68-4bf6-93e6-1e50e22753ea /tidb/cdc/default/default/upstream/7365754086620651211 {"id":7365754086620651211,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5b3cfc9d-9c68-4bf6-93e6-1e50e22753ea {"id":"5b3cfc9d-9c68-4bf6-93e6-1e50e22753ea","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973280} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f413fd6 5b3cfc9d-9c68-4bf6-93e6-1e50e22753ea /tidb/cdc/default/default/upstream/7365754086620651211 {"id":7365754086620651211,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2800.out cli changefeed create --start-ts=449569955118841857 --sink-uri=mysql://normal:123456@127.0.0.1:3306/tidb-txn-mode=pessimistic + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:28:03 GMT < Content-Length: 1303 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92 {"id":"4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973280} /tidb/cdc/default/__cdc_meta__/capture/ea154161-b8c5-4c73-9e61-dae0975f7c79 {"id":"ea154161-b8c5-4c73-9e61-dae0975f7c79","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973274} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f38d431 ea154161-b8c5-4c73-9e61-dae0975f7c79 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f38d45c 4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92 /tidb/cdc/default/default/upstream/7365754067370870491 {"id":7365754067370870491,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92 {"id":"4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973280} /tidb/cdc/default/__cdc_meta__/capture/ea154161-b8c5-4c73-9e61-dae0975f7c79 {"id":"ea154161-b8c5-4c73-9e61-dae0975f7c79","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973274} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f38d431 ea154161-b8c5-4c73-9e61-dae0975f7c79 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f38d45c 4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92 /tidb/cdc/default/default/upstream/7365754067370870491 {"id":7365754067370870491,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92 {"id":"4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973280} /tidb/cdc/default/__cdc_meta__/capture/ea154161-b8c5-4c73-9e61-dae0975f7c79 {"id":"ea154161-b8c5-4c73-9e61-dae0975f7c79","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973274} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f38d431 ea154161-b8c5-4c73-9e61-dae0975f7c79 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f38d45c 4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92 /tidb/cdc/default/default/upstream/7365754067370870491 {"id":7365754067370870491,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "ea154161-b8c5-4c73-9e61-dae0975f7c79 default" | grep id go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httprc v1.0.5 Create changefeed successfully! ID: 01283c7d-009a-4186-a5f4-12a33c169c62 Info: {"upstream_id":7365754086620651211,"namespace":"default","id":"01283c7d-009a-4186-a5f4-12a33c169c62","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/tidb-txn-mode=pessimistic","create_time":"2024-05-06T13:28:03.537966523+08:00","start_ts":449569955118841857,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569955118841857,"checkpoint_ts":449569955118841857,"checkpoint_time":"2024-05-06 13:27:58.499"} PASS table consistent_replicate_ddl.usertable2 not exists for 2-th check, retry later go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 51-th check, retry later table foreign_key.finish_mark not exists for 1-th check, retry later go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 run task successfully check_changefeed_state http://127.0.0.1:2379 4a92993c-a68c-4f38-92aa-f964dd456791 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=4a92993c-a68c-4f38-92aa-f964dd456791 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 4a92993c-a68c-4f38-92aa-f964dd456791 -s + set +x go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/BurntSushi/toml v1.3.2 + info='{ "upstream_id": 7365754063503306118, "namespace": "default", "id": "4a92993c-a68c-4f38-92aa-f964dd456791", "state": "normal", "checkpoint_tso": 449569955523854341, "checkpoint_time": "2024-05-06 13:28:00.044", "error": null }' + echo '{ "upstream_id": 7365754063503306118, "namespace": "default", "id": "4a92993c-a68c-4f38-92aa-f964dd456791", "state": "normal", "checkpoint_tso": 449569955523854341, "checkpoint_time": "2024-05-06 13:28:00.044", "error": null }' { "upstream_id": 7365754063503306118, "namespace": "default", "id": "4a92993c-a68c-4f38-92aa-f964dd456791", "state": "normal", "checkpoint_tso": 449569955523854341, "checkpoint_time": "2024-05-06 13:28:00.044", "error": null } ++ echo '{' '"upstream_id":' 7365754063503306118, '"namespace":' '"default",' '"id":' '"4a92993c-a68c-4f38-92aa-f964dd456791",' '"state":' '"normal",' '"checkpoint_tso":' 449569955523854341, '"checkpoint_time":' '"2024-05-06' '13:28:00.044",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365754063503306118, '"namespace":' '"default",' '"id":' '"4a92993c-a68c-4f38-92aa-f964dd456791",' '"state":' '"normal",' '"checkpoint_tso":' 449569955523854341, '"checkpoint_time":' '"2024-05-06' '13:28:00.044",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7365754063503306118 449569955523854340 449569955523854341 go: downloading go.uber.org/atomic v1.11.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda "id": "4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92", run task successfully go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda + set +x go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading go.uber.org/zap v1.27.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading golang.org/x/text v0.14.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/google/uuid v1.6.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/docker/go-units v0.5.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/google/btree v1.1.2 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec table consistent_replicate_ddl.usertable2 exists table consistent_replicate_ddl.usertable3 not exists for 1-th check, retry later go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda table foreign_key.finish_mark not exists for 2-th check, retry later run task successfully check_changefeed_state http://127.0.0.1:2379 4a92993c-a68c-4f38-92aa-f964dd456791 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=4a92993c-a68c-4f38-92aa-f964dd456791 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 4a92993c-a68c-4f38-92aa-f964dd456791 -s go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading golang.org/x/sys v0.19.0 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda + info='{ "upstream_id": 7365754063503306118, "namespace": "default", "id": "4a92993c-a68c-4f38-92aa-f964dd456791", "state": "stopped", "checkpoint_tso": 449569955523854341, "checkpoint_time": "2024-05-06 13:28:00.044", "error": null }' + echo '{ "upstream_id": 7365754063503306118, "namespace": "default", "id": "4a92993c-a68c-4f38-92aa-f964dd456791", "state": "stopped", "checkpoint_tso": 449569955523854341, "checkpoint_time": "2024-05-06 13:28:00.044", "error": null }' { "upstream_id": 7365754063503306118, "namespace": "default", "id": "4a92993c-a68c-4f38-92aa-f964dd456791", "state": "stopped", "checkpoint_tso": 449569955523854341, "checkpoint_time": "2024-05-06 13:28:00.044", "error": null } ++ echo '{' '"upstream_id":' 7365754063503306118, '"namespace":' '"default",' '"id":' '"4a92993c-a68c-4f38-92aa-f964dd456791",' '"state":' '"stopped",' '"checkpoint_tso":' 449569955523854341, '"checkpoint_time":' '"2024-05-06' '13:28:00.044",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365754063503306118, '"namespace":' '"default",' '"id":' '"4a92993c-a68c-4f38-92aa-f964dd456791",' '"state":' '"stopped",' '"checkpoint_tso":' 449569955523854341, '"checkpoint_time":' '"2024-05-06' '13:28:00.044",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully capture_id: 4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92 pass test: check health table ddl_manager.finish_mark not exists for 52-th check, retry later check_changefeed_state http://127.0.0.1:2379 1c7a7eb3-9b2f-480e-8865-6a95b7085c0a normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=1c7a7eb3-9b2f-480e-8865-6a95b7085c0a + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 1c7a7eb3-9b2f-480e-8865-6a95b7085c0a -s + info='{ "upstream_id": 7365754063503306118, "namespace": "default", "id": "1c7a7eb3-9b2f-480e-8865-6a95b7085c0a", "state": "normal", "checkpoint_tso": 449569957109825540, "checkpoint_time": "2024-05-06 13:28:06.094", "error": null }' + echo '{ "upstream_id": 7365754063503306118, "namespace": "default", "id": "1c7a7eb3-9b2f-480e-8865-6a95b7085c0a", "state": "normal", "checkpoint_tso": 449569957109825540, "checkpoint_time": "2024-05-06 13:28:06.094", "error": null }' { "upstream_id": 7365754063503306118, "namespace": "default", "id": "1c7a7eb3-9b2f-480e-8865-6a95b7085c0a", "state": "normal", "checkpoint_tso": 449569957109825540, "checkpoint_time": "2024-05-06 13:28:06.094", "error": null } ++ echo '{' '"upstream_id":' 7365754063503306118, '"namespace":' '"default",' '"id":' '"1c7a7eb3-9b2f-480e-8865-6a95b7085c0a",' '"state":' '"normal",' '"checkpoint_tso":' 449569957109825540, '"checkpoint_time":' '"2024-05-06' '13:28:06.094",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365754063503306118, '"namespace":' '"default",' '"id":' '"1c7a7eb3-9b2f-480e-8865-6a95b7085c0a",' '"state":' '"normal",' '"checkpoint_tso":' 449569957109825540, '"checkpoint_time":' '"2024-05-06' '13:28:06.094",' '"error":' null '}' ++ jq -r .error.message go: downloading golang.org/x/text v0.14.0 pass test: get status + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365754063503306118 pass test: create changefeed go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/google/btree v1.1.2 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/google/uuid v1.6.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading cloud.google.com/go v0.112.2 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 table consistent_replicate_ddl.usertable3 not exists for 2-th check, retry later go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 table foreign_key.finish_mark not exists for 3-th check, retry later go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 table ddl_manager.finish_mark not exists for 53-th check, retry later go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/robfig/cron v1.2.0 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading golang.org/x/oauth2 v0.18.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/ncw/directio v1.0.5 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 table foreign_key.finish_mark not exists for 4-th check, retry later go: downloading k8s.io/apimachinery v0.28.6 table consistent_replicate_ddl.usertable3 exists table consistent_replicate_ddl.usertable_bak not exists for 1-th check, retry later go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b run task successfully Changefeed remove successfully. ID: 4a92993c-a68c-4f38-92aa-f964dd456791 CheckpointTs: 449569955523854341 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 table ddl_manager.finish_mark not exists for 54-th check, retry later check_safepoint_forward http://127.0.0.1:2379 7365754063503306118 449569957109825539 449569957109825540 449569955523854341 table row_format.finish_mark not exists for 1-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 23.71 secs (157319685 bytes/sec) [Pipeline] { [Pipeline] cache run task successfully table test.simple1 not exists for 1-th check, retry later Changefeed remove successfully. ID: 1c7a7eb3-9b2f-480e-8865-6a95b7085c0a CheckpointTs: 449569958145294341 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_cleared http://127.0.0.1:2379 7365754063503306118 run task successfully table row_format.finish_mark not exists for 2-th check, retry later table consistent_replicate_ddl.usertable_bak not exists for 2-th check, retry later go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/felixge/httpsnoop v1.0.4 table ddl_manager.finish_mark not exists for 55-th check, retry later table foreign_key.finish_mark not exists for 5-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table test.simple1 exists cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:28:13 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd table consistent_replicate_ddl.usertable_bak exists pass test: list changefeed table ddl_manager.finish_mark not exists for 56-th check, retry later table foreign_key.finish_mark not exists for 6-th check, retry later pass test: get changefeed table row_format.finish_mark not exists for 3-th check, retry later pass test: pause changefeed pass test: update changefeed wait process 2716 exit for 1-th time... table ddl_manager.finish_mark not exists for 57-th check, retry later table foreign_key.finish_mark not exists for 7-th check, retry later table row_format.finish_mark not exists for 4-th check, retry later wait process 2716 exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... wait process 2716 exit for 3-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (2716) - No such process wait process 2716 exit for 4-th time... process 2716 already exit cdc.test cli capture list --server http://127.0.0.1:8301 2>&1 |grep 4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92 -A1 | grep '"is-owner": true' table foreign_key.finish_mark not exists for 8-th check, retry later table row_format.finish_mark not exists for 5-th check, retry later table ddl_manager.finish_mark not exists for 58-th check, retry later "is-owner": true, run task successfully [Mon May 6 13:28:18 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.31223124.out server --log-file /tmp/tidb_cdc_test/http_api/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... Verifying downstream PD is started... wait process cdc.test exit for 2-th time... table foreign_key.finish_mark not exists for 9-th check, retry later table row_format.finish_mark not exists for 6-th check, retry later wait process cdc.test exit for 3-th time... table ddl_manager.finish_mark not exists for 59-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:28:20 CST 2024] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true);github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.28752877.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server2 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table row_format.finish_mark not exists for 7-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:28:21 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/changefeed-test1 {UpstreamID:7365754067370870491 Namespace:default ID:changefeed-test1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:28:06.539717543 +0800 CST StartTs:449569957202100234 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00176a900 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569957214945290} {CheckpointTs:449569957739495456 MinTableBarrierTs:449569960937390091 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449569960937390090, checkpointTs: 449569960937390090, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449569960937390090, checkpointTs: 449569960937390090, state: Replicating changefeedID: default/changefeed-test3 {UpstreamID:7365754067370870491 Namespace:default ID:changefeed-test3 SinkURI:blackhole:// CreateTime:2024-05-06 13:28:06.747802813 +0800 CST StartTs:449569957267374092 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00259ccf0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569957280743425} {CheckpointTs:449569957739495456 MinTableBarrierTs:449569960937390091 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92 {"id":"4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973280} /tidb/cdc/default/__cdc_meta__/capture/cf8a6922-4fd0-45a3-ad3d-d39689156916 {"id":"cf8a6922-4fd0-45a3-ad3d-d39689156916","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973298} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f38d45c 4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f38d4e4 cf8a6922-4fd0-45a3-ad3d-d39689156916 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7365754067370870491,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:28:06.539717543+08:00","start-ts":449569957202100234,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569957214945290} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7365754067370870491,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-05-06T13:28:06.648067666+08:00","start-ts":449569957241159689,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"stopped","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569959272775683} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7365754067370870491,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-05-06T13:28:06.747802813+08:00","start-ts":449569957267374092,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569957280743425} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":449569957739495456,"min-table-barrier-ts":449569960937390091,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":449569957739495456,"min-table-barrier-ts":449569959102382091,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":449569957739495456,"min-table-barrier-ts":449569960937390091,"admin-job-type":0} /tidb/cdc/default/default/task/position/4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cf8a6922-4fd0-45a3-ad3d-d39689156916/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cf8a6922-4fd0-45a3-ad3d-d39689156916/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754067370870491 {"id":7365754067370870491,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/changefeed-test1 {UpstreamID:7365754067370870491 Namespace:default ID:changefeed-test1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:28:06.539717543 +0800 CST StartTs:449569957202100234 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00176a900 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569957214945290} {CheckpointTs:449569957739495456 MinTableBarrierTs:449569960937390091 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449569960937390090, checkpointTs: 449569960937390090, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449569960937390090, checkpointTs: 449569960937390090, state: Replicating changefeedID: default/changefeed-test3 {UpstreamID:7365754067370870491 Namespace:default ID:changefeed-test3 SinkURI:blackhole:// CreateTime:2024-05-06 13:28:06.747802813 +0800 CST StartTs:449569957267374092 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00259ccf0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569957280743425} {CheckpointTs:449569957739495456 MinTableBarrierTs:449569960937390091 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92 {"id":"4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973280} /tidb/cdc/default/__cdc_meta__/capture/cf8a6922-4fd0-45a3-ad3d-d39689156916 {"id":"cf8a6922-4fd0-45a3-ad3d-d39689156916","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973298} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f38d45c 4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f38d4e4 cf8a6922-4fd0-45a3-ad3d-d39689156916 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7365754067370870491,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:28:06.539717543+08:00","start-ts":449569957202100234,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569957214945290} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7365754067370870491,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-05-06T13:28:06.648067666+08:00","start-ts":449569957241159689,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"stopped","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569959272775683} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7365754067370870491,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-05-06T13:28:06.747802813+08:00","start-ts":449569957267374092,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569957280743425} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":449569957739495456,"min-table-barrier-ts":449569960937390091,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":449569957739495456,"min-table-barrier-ts":449569959102382091,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":449569957739495456,"min-table-barrier-ts":449569960937390091,"admin-job-type":0} /tidb/cdc/default/default/task/position/4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cf8a6922-4fd0-45a3-ad3d-d39689156916/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cf8a6922-4fd0-45a3-ad3d-d39689156916/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754067370870491 {"id":7365754067370870491,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/changefeed-test1 {UpstreamID:7365754067370870491 Namespace:default ID:changefeed-test1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:28:06.539717543 +0800 CST StartTs:449569957202100234 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00176a900 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569957214945290} {CheckpointTs:449569957739495456 MinTableBarrierTs:449569960937390091 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449569960937390090, checkpointTs: 449569960937390090, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449569960937390090, checkpointTs: 449569960937390090, state: Replicating changefeedID: default/changefeed-test3 {UpstreamID:7365754067370870491 Namespace:default ID:changefeed-test3 SinkURI:blackhole:// CreateTime:2024-05-06 13:28:06.747802813 +0800 CST StartTs:449569957267374092 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00259ccf0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569957280743425} {CheckpointTs:449569957739495456 MinTableBarrierTs:449569960937390091 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92 {"id":"4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973280} /tidb/cdc/default/__cdc_meta__/capture/cf8a6922-4fd0-45a3-ad3d-d39689156916 {"id":"cf8a6922-4fd0-45a3-ad3d-d39689156916","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973298} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f38d45c 4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f38d4e4 cf8a6922-4fd0-45a3-ad3d-d39689156916 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7365754067370870491,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:28:06.539717543+08:00","start-ts":449569957202100234,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569957214945290} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7365754067370870491,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-05-06T13:28:06.648067666+08:00","start-ts":449569957241159689,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"stopped","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569959272775683} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7365754067370870491,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-05-06T13:28:06.747802813+08:00","start-ts":449569957267374092,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569957280743425} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":449569957739495456,"min-table-barrier-ts":449569960937390091,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":449569957739495456,"min-table-barrier-ts":449569959102382091,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":449569957739495456,"min-table-barrier-ts":449569960937390091,"admin-job-type":0} /tidb/cdc/default/default/task/position/4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/4f9066bc-84e8-4e1f-b8c1-a6a84dcb3e92/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cf8a6922-4fd0-45a3-ad3d-d39689156916/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cf8a6922-4fd0-45a3-ad3d-d39689156916/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754067370870491 {"id":7365754067370870491,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"address": "127.0.0.1:8300"' table ddl_manager.finish_mark not exists for 60-th check, retry later table foreign_key.finish_mark not exists for 10-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... "address": "127.0.0.1:8300", run task successfully ***************** properties ***************** "mysql.db"="testSync" "mysql.port"="4000" "recordcount"="10" "dotransactions"="false" "threadcount"="2" "scanproportion"="0" "operationcount"="0" "mysql.host"="127.0.0.1" "workload"="core" "insertproportion"="0" "readallfields"="true" "readproportion"="0" "updateproportion"="0" "requestdistribution"="uniform" "mysql.user"="root" ********************************************** Run finished, takes 6.380669ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2459.1, Avg(us): 1106, Min(us): 533, Max(us): 3695, 95th(us): 4000, 99th(us): 4000 pass test: list changefeed + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:28:23 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/1e437b4f-cab5-4697-9336-f86b8189cf0a {UpstreamID:7365754062463967314 Namespace:default ID:1e437b4f-cab5-4697-9336-f86b8189cf0a SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:27:56.037000142 +0800 CST StartTs:449569954434646023 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0024b07e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569954460598281} {CheckpointTs:449569960083587079 MinTableBarrierTs:449569961407414273 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449569961407414273, checkpointTs: 449569961407414273, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449569961407414273, checkpointTs: 449569961407414273, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449569961407414273, checkpointTs: 449569961407414273, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449569961407414273, checkpointTs: 449569961407414273, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449569961407414273, checkpointTs: 449569961407414273, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9fd75d3e-1a61-414d-b2a5-c4124118e297 {"id":"9fd75d3e-1a61-414d-b2a5-c4124118e297","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973300} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f2c500d 9fd75d3e-1a61-414d-b2a5-c4124118e297 /tidb/cdc/default/default/changefeed/info/1e437b4f-cab5-4697-9336-f86b8189cf0a {"upstream-id":7365754062463967314,"namespace":"default","changefeed-id":"1e437b4f-cab5-4697-9336-f86b8189cf0a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:27:56.037000142+08:00","start-ts":449569954434646023,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569954460598281} /tidb/cdc/default/default/changefeed/status/1e437b4f-cab5-4697-9336-f86b8189cf0a {"checkpoint-ts":449569961407414273,"min-table-barrier-ts":449569961407414273,"admin-job-type":0} /tidb/cdc/default/default/task/position/9fd75d3e-1a61-414d-b2a5-c4124118e297/1e437b4f-cab5-4697-9336-f86b8189cf0a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754062463967314 {"id":7365754062463967314,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/1e437b4f-cab5-4697-9336-f86b8189cf0a {UpstreamID:7365754062463967314 Namespace:default ID:1e437b4f-cab5-4697-9336-f86b8189cf0a SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:27:56.037000142 +0800 CST StartTs:449569954434646023 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0024b07e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569954460598281} {CheckpointTs:449569960083587079 MinTableBarrierTs:449569961407414273 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449569961407414273, checkpointTs: 449569961407414273, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449569961407414273, checkpointTs: 449569961407414273, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449569961407414273, checkpointTs: 449569961407414273, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449569961407414273, checkpointTs: 449569961407414273, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449569961407414273, checkpointTs: 449569961407414273, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9fd75d3e-1a61-414d-b2a5-c4124118e297 {"id":"9fd75d3e-1a61-414d-b2a5-c4124118e297","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973300} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f2c500d 9fd75d3e-1a61-414d-b2a5-c4124118e297 /tidb/cdc/default/default/changefeed/info/1e437b4f-cab5-4697-9336-f86b8189cf0a {"upstream-id":7365754062463967314,"namespace":"default","changefeed-id":"1e437b4f-cab5-4697-9336-f86b8189cf0a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:27:56.037000142+08:00","start-ts":449569954434646023,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569954460598281} /tidb/cdc/default/default/changefeed/status/1e437b4f-cab5-4697-9336-f86b8189cf0a {"checkpoint-ts":449569961407414273,"min-table-barrier-ts":449569961407414273,"admin-job-type":0} /tidb/cdc/default/default/task/position/9fd75d3e-1a61-414d-b2a5-c4124118e297/1e437b4f-cab5-4697-9336-f86b8189cf0a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754062463967314 {"id":7365754062463967314,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/1e437b4f-cab5-4697-9336-f86b8189cf0a {UpstreamID:7365754062463967314 Namespace:default ID:1e437b4f-cab5-4697-9336-f86b8189cf0a SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:27:56.037000142 +0800 CST StartTs:449569954434646023 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0024b07e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569954460598281} {CheckpointTs:449569960083587079 MinTableBarrierTs:449569961407414273 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449569961407414273, checkpointTs: 449569961407414273, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449569961407414273, checkpointTs: 449569961407414273, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449569961407414273, checkpointTs: 449569961407414273, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449569961407414273, checkpointTs: 449569961407414273, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449569961407414273, checkpointTs: 449569961407414273, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9fd75d3e-1a61-414d-b2a5-c4124118e297 {"id":"9fd75d3e-1a61-414d-b2a5-c4124118e297","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973300} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5f2c500d 9fd75d3e-1a61-414d-b2a5-c4124118e297 /tidb/cdc/default/default/changefeed/info/1e437b4f-cab5-4697-9336-f86b8189cf0a {"upstream-id":7365754062463967314,"namespace":"default","changefeed-id":"1e437b4f-cab5-4697-9336-f86b8189cf0a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:27:56.037000142+08:00","start-ts":449569954434646023,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569954460598281} /tidb/cdc/default/default/changefeed/status/1e437b4f-cab5-4697-9336-f86b8189cf0a {"checkpoint-ts":449569961407414273,"min-table-barrier-ts":449569961407414273,"admin-job-type":0} /tidb/cdc/default/default/task/position/9fd75d3e-1a61-414d-b2a5-c4124118e297/1e437b4f-cab5-4697-9336-f86b8189cf0a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754062463967314 {"id":7365754062463967314,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 61-th check, retry later table foreign_key.finish_mark not exists for 11-th check, retry later pass test: get changefeed table row_format.finish_mark not exists for 8-th check, retry later pass test: pause changefeed ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: resume changefeed pass test: rebalance table pass test: move table table foreign_key.finish_mark not exists for 12-th check, retry later table row_format.finish_mark not exists for 9-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 62-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table foreign_key.finish_mark exists check diff successfully table row_format.finish_mark not exists for 10-th check, retry later wait process cdc.test exit for 1-th time... pass test: get processors table ddl_manager.finish_mark not exists for 63-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: list processors pass test: set log level wait process cdc.test exit for 2-th time... table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "readallfields"="true" "threadcount"="2" "readproportion"="0" "dotransactions"="false" "mysql.user"="root" "requestdistribution"="uniform" "workload"="core" "updateproportion"="0" "mysql.port"="4000" "mysql.db"="sink_retry" "mysql.host"="127.0.0.1" "scanproportion"="0" "recordcount"="10" "insertproportion"="0" "operationcount"="0" ********************************************** Run finished, takes 3.985678ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3427.7, Avg(us): 701, Min(us): 407, Max(us): 1548, 95th(us): 2000, 99th(us): 2000 pass test: remove changefeed Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 15.84 secs (235511390 bytes/sec) [Pipeline] { [Pipeline] cache pass test: resign owner wait process cdc.test exit for 3-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table row_format.finish_mark exists cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:28:29 CST 2024] <<<<<< run test case foreign_key success! >>>>>> check diff successfully table ddl_manager.finish_mark not exists for 64-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317fcda40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57, pid:4022, start at 2024-05-06 13:28:27.796272654 +0800 CST m=+5.566538417 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:30:27.804 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:28:27.803 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:18:27.803 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317fcda40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57, pid:4022, start at 2024-05-06 13:28:27.796272654 +0800 CST m=+5.566538417 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:30:27.804 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:28:27.803 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:18:27.803 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d317fce5c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57, pid:4099, start at 2024-05-06 13:28:27.837979368 +0800 CST m=+5.549553908 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:30:27.845 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:28:27.849 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:18:27.849 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:28:31 CST 2024] <<<<<< run test case row_format success! >>>>>> table ddl_manager.finish_mark not exists for 65-th check, retry later [Mon May 6 13:28:32 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.54085410.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:28:33 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 66-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:28:35 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12e450fb-907d-425d-9a15-136e8d9466f4 {"id":"12e450fb-907d-425d-9a15-136e8d9466f4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973312} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5fc5e3d0 12e450fb-907d-425d-9a15-136e8d9466f4 /tidb/cdc/default/default/upstream/7365754227330603082 {"id":7365754227330603082,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12e450fb-907d-425d-9a15-136e8d9466f4 {"id":"12e450fb-907d-425d-9a15-136e8d9466f4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973312} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5fc5e3d0 12e450fb-907d-425d-9a15-136e8d9466f4 /tidb/cdc/default/default/upstream/7365754227330603082 {"id":7365754227330603082,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12e450fb-907d-425d-9a15-136e8d9466f4 {"id":"12e450fb-907d-425d-9a15-136e8d9466f4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973312} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5fc5e3d0 12e450fb-907d-425d-9a15-136e8d9466f4 /tidb/cdc/default/default/upstream/7365754227330603082 {"id":7365754227330603082,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 67-th check, retry later ***************** properties ***************** "mysql.port"="4000" "readproportion"="0" "mysql.host"="127.0.0.1" "readallfields"="true" "operationcount"="0" "updateproportion"="0" "recordcount"="50" "mysql.user"="root" "threadcount"="4" "workload"="core" "insertproportion"="0" "mysql.db"="changefeed_reconstruct" "scanproportion"="0" "dotransactions"="false" "requestdistribution"="uniform" ********************************************** Run finished, takes 19.898554ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3203.0, Avg(us): 1589, Min(us): 1049, Max(us): 4935, 95th(us): 5000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3180534c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd, pid:4557, start at 2024-05-06 13:28:36.351293177 +0800 CST m=+5.153817786 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:30:36.357 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:28:36.358 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:18:36.358 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3180534c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd, pid:4557, start at 2024-05-06 13:28:36.351293177 +0800 CST m=+5.153817786 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:30:36.357 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:28:36.358 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:18:36.358 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318055940003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd, pid:4641, start at 2024-05-06 13:28:36.454327822 +0800 CST m=+5.202688332 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:30:36.460 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:28:36.453 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:18:36.453 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 68-th check, retry later table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later [Mon May 6 13:28:39 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.59015903.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 69-th check, retry later check diff successfully wait process 5413 exit for 1-th time... table ddl_manager.finish_mark not exists for 70-th check, retry later wait process 5413 exit for 2-th time... wait process 5413 exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:28:42 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dfdafe91-db7d-4e97-9b2c-a5cbf9166d72 {"id":"dfdafe91-db7d-4e97-9b2c-a5cbf9166d72","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973319} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5fecf7c6 dfdafe91-db7d-4e97-9b2c-a5cbf9166d72 /tidb/cdc/default/default/upstream/7365754272009099929 {"id":7365754272009099929,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dfdafe91-db7d-4e97-9b2c-a5cbf9166d72 {"id":"dfdafe91-db7d-4e97-9b2c-a5cbf9166d72","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973319} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5fecf7c6 dfdafe91-db7d-4e97-9b2c-a5cbf9166d72 /tidb/cdc/default/default/upstream/7365754272009099929 {"id":7365754272009099929,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dfdafe91-db7d-4e97-9b2c-a5cbf9166d72 {"id":"dfdafe91-db7d-4e97-9b2c-a5cbf9166d72","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973319} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5fecf7c6 dfdafe91-db7d-4e97-9b2c-a5cbf9166d72 /tidb/cdc/default/default/upstream/7365754272009099929 {"id":7365754272009099929,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: mysql... <<================= /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5413) - No such process wait process 5413 exit for 4-th time... process 5413 already exit check_no_capture http://127.0.0.1:2379 table changefeed_pause_resume.t1 not exists for 1-th check, retry later parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 13:28:43 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.56845686.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 71-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 13.17 secs (283145181 bytes/sec) [Pipeline] { [Pipeline] cache table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 72-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:28:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e03cd9db-a6e4-4374-acac-163796f86718 {UpstreamID:7365754227330603082 Namespace:default ID:e03cd9db-a6e4-4374-acac-163796f86718 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:28:35.870146153 +0800 CST StartTs:449569964884754441 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00335fd40 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569964910968835} {CheckpointTs:449569966077509635 MinTableBarrierTs:449569967650373636 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449569966077509635, checkpointTs: 449569966077509635, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0156014d-6645-4c57-be02-ea52438ff496 {"id":"0156014d-6645-4c57-be02-ea52438ff496","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973323} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5fc5e478 0156014d-6645-4c57-be02-ea52438ff496 /tidb/cdc/default/default/changefeed/info/e03cd9db-a6e4-4374-acac-163796f86718 {"upstream-id":7365754227330603082,"namespace":"default","changefeed-id":"e03cd9db-a6e4-4374-acac-163796f86718","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:28:35.870146153+08:00","start-ts":449569964884754441,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569964910968835} /tidb/cdc/default/default/changefeed/status/e03cd9db-a6e4-4374-acac-163796f86718 {"checkpoint-ts":449569966077509635,"min-table-barrier-ts":449569967650373636,"admin-job-type":0} /tidb/cdc/default/default/task/position/0156014d-6645-4c57-be02-ea52438ff496/e03cd9db-a6e4-4374-acac-163796f86718 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754227330603082 {"id":7365754227330603082,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e03cd9db-a6e4-4374-acac-163796f86718 {UpstreamID:7365754227330603082 Namespace:default ID:e03cd9db-a6e4-4374-acac-163796f86718 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:28:35.870146153 +0800 CST StartTs:449569964884754441 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00335fd40 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569964910968835} {CheckpointTs:449569966077509635 MinTableBarrierTs:449569967650373636 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449569966077509635, checkpointTs: 449569966077509635, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0156014d-6645-4c57-be02-ea52438ff496 {"id":"0156014d-6645-4c57-be02-ea52438ff496","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973323} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5fc5e478 0156014d-6645-4c57-be02-ea52438ff496 /tidb/cdc/default/default/changefeed/info/e03cd9db-a6e4-4374-acac-163796f86718 {"upstream-id":7365754227330603082,"namespace":"default","changefeed-id":"e03cd9db-a6e4-4374-acac-163796f86718","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:28:35.870146153+08:00","start-ts":449569964884754441,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569964910968835} /tidb/cdc/default/default/changefeed/status/e03cd9db-a6e4-4374-acac-163796f86718 {"checkpoint-ts":449569966077509635,"min-table-barrier-ts":449569967650373636,"admin-job-type":0} /tidb/cdc/default/default/task/position/0156014d-6645-4c57-be02-ea52438ff496/e03cd9db-a6e4-4374-acac-163796f86718 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754227330603082 + grep -q 'failed to get info:' {"id":7365754227330603082,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e03cd9db-a6e4-4374-acac-163796f86718 {UpstreamID:7365754227330603082 Namespace:default ID:e03cd9db-a6e4-4374-acac-163796f86718 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:28:35.870146153 +0800 CST StartTs:449569964884754441 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00335fd40 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569964910968835} {CheckpointTs:449569966077509635 MinTableBarrierTs:449569967650373636 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449569966077509635, checkpointTs: 449569966077509635, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0156014d-6645-4c57-be02-ea52438ff496 {"id":"0156014d-6645-4c57-be02-ea52438ff496","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973323} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c5fc5e478 0156014d-6645-4c57-be02-ea52438ff496 /tidb/cdc/default/default/changefeed/info/e03cd9db-a6e4-4374-acac-163796f86718 {"upstream-id":7365754227330603082,"namespace":"default","changefeed-id":"e03cd9db-a6e4-4374-acac-163796f86718","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:28:35.870146153+08:00","start-ts":449569964884754441,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569964910968835} /tidb/cdc/default/default/changefeed/status/e03cd9db-a6e4-4374-acac-163796f86718 {"checkpoint-ts":449569966077509635,"min-table-barrier-ts":449569967650373636,"admin-job-type":0} /tidb/cdc/default/default/task/position/0156014d-6645-4c57-be02-ea52438ff496/e03cd9db-a6e4-4374-acac-163796f86718 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754227330603082 {"id":7365754227330603082,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... "id": "0156014d-6645-4c57-be02-ea52438ff496", "cluster-id": "default" run task successfully table ddl_manager.finish_mark not exists for 73-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) capture_id: 0156014d-6645-4c57-be02-ea52438ff496 check_processor_table_count http://127.0.0.1:2379 e03cd9db-a6e4-4374-acac-163796f86718 0156014d-6645-4c57-be02-ea52438ff496 1 table changefeed_pause_resume.t3 exists start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... run task successfully Verifying downstream PD is started... table ddl_manager.finish_mark not exists for 74-th check, retry later check_processor_table_count http://127.0.0.1:2379 e03cd9db-a6e4-4374-acac-163796f86718 0156014d-6645-4c57-be02-ea52438ff496 0 table count 1 does equal to expected count 0 run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 75-th check, retry later check_processor_table_count http://127.0.0.1:2379 e03cd9db-a6e4-4374-acac-163796f86718 0156014d-6645-4c57-be02-ea52438ff496 0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run task successfully Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 76-th check, retry later ***************** properties ***************** "scanproportion"="0" "dotransactions"="false" "recordcount"="50" "operationcount"="0" "updateproportion"="0" "readproportion"="0" "readallfields"="true" "threadcount"="4" "workload"="core" "requestdistribution"="uniform" "mysql.db"="changefeed_reconstruct" "insertproportion"="0" "mysql.port"="4000" "mysql.user"="root" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 19.869285ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3247.3, Avg(us): 1571, Min(us): 1045, Max(us): 5241, 95th(us): 5000, 99th(us): 6000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... check diff failed 3-th time, retry later Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318161fc0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54, pid:3800, start at 2024-05-06 13:28:53.663756934 +0800 CST m=+5.239990592 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:30:53.671 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:28:53.631 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:18:53.631 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318161fc0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54, pid:3800, start at 2024-05-06 13:28:53.663756934 +0800 CST m=+5.239990592 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:30:53.671 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:28:53.631 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:18:53.631 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318163740014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54, pid:3866, start at 2024-05-06 13:28:53.746970851 +0800 CST m=+5.271926327 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:30:53.754 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:28:53.725 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:18:53.725 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table ddl_manager.finish_mark not exists for 77-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 78-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5171.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:28:59 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> table ddl_manager.finish_mark not exists for 79-th check, retry later + set +x + tso='449569970986942465 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449569970986942465 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:28:59 CST 2024] <<<<<< run test case sink_retry success! >>>>>> pass test: get tso wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3181ac740006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3, pid:3924, start at 2024-05-06 13:28:58.404167899 +0800 CST m=+5.310288868 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:30:58.411 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:28:58.397 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:18:58.397 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3181ac740006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3, pid:3924, start at 2024-05-06 13:28:58.404167899 +0800 CST m=+5.310288868 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:30:58.411 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:28:58.397 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:18:58.397 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3181ae380015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3, pid:3990, start at 2024-05-06 13:28:58.55771836 +0800 CST m=+5.411514809 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:30:58.566 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:28:58.560 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:18:58.560 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:29:01 CST 2024] <<<<<< run test case http_api success! >>>>>> [Mon May 6 13:29:01 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.52285230.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3181d020000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-xwksv-n7b01-mfk0q, pid:4178, start at 2024-05-06 13:29:00.695949398 +0800 CST m=+5.323540156 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:00.707 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:00.680 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:00.680 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 80-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5257.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 1-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 16.43 secs (227083680 bytes/sec) [Pipeline] { [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3181d020000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-xwksv-n7b01-mfk0q, pid:4178, start at 2024-05-06 13:29:00.695949398 +0800 CST m=+5.323540156 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:00.707 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:00.680 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:00.680 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3181d24c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-xwksv-n7b01-mfk0q, pid:4262, start at 2024-05-06 13:29:00.849678285 +0800 CST m=+5.377755842 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:00.856 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:00.819 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:00.819 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:29:04 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3b89af25-faaf-4f30-aef1-0ea019bc91a3 {"id":"3b89af25-faaf-4f30-aef1-0ea019bc91a3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973341} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c602e49f4 3b89af25-faaf-4f30-aef1-0ea019bc91a3 /tidb/cdc/default/default/upstream/7365754347266982793 {"id":7365754347266982793,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3b89af25-faaf-4f30-aef1-0ea019bc91a3 {"id":"3b89af25-faaf-4f30-aef1-0ea019bc91a3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973341} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c602e49f4 3b89af25-faaf-4f30-aef1-0ea019bc91a3 /tidb/cdc/default/default/upstream/7365754347266982793 {"id":7365754347266982793,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3b89af25-faaf-4f30-aef1-0ea019bc91a3 {"id":"3b89af25-faaf-4f30-aef1-0ea019bc91a3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973341} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c602e49f4 3b89af25-faaf-4f30-aef1-0ea019bc91a3 /tidb/cdc/default/default/upstream/7365754347266982793 {"id":7365754347266982793,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5288.out cli changefeed create --start-ts=449569970986942465 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: af92a2ab-9830-4465-b30b-256722ef66f4 Info: {"upstream_id":7365754347266982793,"namespace":"default","id":"af92a2ab-9830-4465-b30b-256722ef66f4","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:29:05.113592538+08:00","start_ts":449569970986942465,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569970986942465,"checkpoint_ts":449569970986942465,"checkpoint_time":"2024-05-06 13:28:59.031"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + tso='449569972210106369 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449569972210106369 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:29:05 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.52955297.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... [Mon May 6 13:29:05 CST 2024] <<<<<< START cdc server in cdc case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.55095511.out server --log-file /tmp/tidb_cdc_test/cdc/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:29:06 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> + set +x check diff failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:29:08 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e36fe793-eddb-4387-bcf5-3a3b37598d42 {"id":"e36fe793-eddb-4387-bcf5-3a3b37598d42","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973345} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c604053d5 e36fe793-eddb-4387-bcf5-3a3b37598d42 /tidb/cdc/default/default/upstream/7365754360054250302 {"id":7365754360054250302,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e36fe793-eddb-4387-bcf5-3a3b37598d42 {"id":"e36fe793-eddb-4387-bcf5-3a3b37598d42","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973345} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c604053d5 e36fe793-eddb-4387-bcf5-3a3b37598d42 /tidb/cdc/default/default/upstream/7365754360054250302 {"id":7365754360054250302,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e36fe793-eddb-4387-bcf5-3a3b37598d42 {"id":"e36fe793-eddb-4387-bcf5-3a3b37598d42","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973345} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c604053d5 e36fe793-eddb-4387-bcf5-3a3b37598d42 /tidb/cdc/default/default/upstream/7365754360054250302 {"id":7365754360054250302,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5353.out cli changefeed create --start-ts=449569972210106369 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 365feecc-3e64-44a4-bb68-173aca682708 Info: {"upstream_id":7365754360054250302,"namespace":"default","id":"365feecc-3e64-44a4-bb68-173aca682708","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:29:08.720933477+08:00","start_ts":449569972210106369,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569972210106369,"checkpoint_ts":449569972210106369,"checkpoint_time":"2024-05-06 13:29:03.697"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:29:08 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2586af16-4fdf-4376-a479-9518d98aba22 {"id":"2586af16-4fdf-4376-a479-9518d98aba22","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973346} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60496ecd 2586af16-4fdf-4376-a479-9518d98aba22 /tidb/cdc/default/default/upstream/7365754378312855759 {"id":7365754378312855759,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2586af16-4fdf-4376-a479-9518d98aba22 {"id":"2586af16-4fdf-4376-a479-9518d98aba22","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973346} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60496ecd 2586af16-4fdf-4376-a479-9518d98aba22 /tidb/cdc/default/default/upstream/7365754378312855759 {"id":7365754378312855759,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2586af16-4fdf-4376-a479-9518d98aba22 {"id":"2586af16-4fdf-4376-a479-9518d98aba22","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973346} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60496ecd 2586af16-4fdf-4376-a479-9518d98aba22 /tidb/cdc/default/default/upstream/7365754378312855759 {"id":7365754378312855759,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.5560.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config /tmp/tidb_cdc_test/cdc/pulsar_test.toml =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Create changefeed successfully! ID: 07eebf80-0f33-4cf0-93d7-8cb3d2f9d22b Info: {"upstream_id":7365754378312855759,"namespace":"default","id":"07eebf80-0f33-4cf0-93d7-8cb3d2f9d22b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:29:09.263418442+08:00","start_ts":449569973647441922,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569973647441922,"checkpoint_ts":449569973647441922,"checkpoint_time":"2024-05-06 13:29:09.180"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff failed 4-th time, retry later + set +x go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/text v0.14.0 check diff failed 5-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/http_api_tls/run.sh using Sink-Type: mysql... <<================= WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Requirement already up-to-date: requests==2.26.0 in /usr/local/lib/python3.6/site-packages Requirement already up-to-date: certifi>=2017.4.17 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: idna<4,>=2.5; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + set +x go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/gorilla/mux v1.8.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading cloud.google.com/go v0.112.2 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/jfcg/sixb v1.3.8 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/kr/pretty v0.3.1 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 check diff successfully go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: mysql... <<================= Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table drop_tables.c not exists for 1-th check, retry later check diff failed 2-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 table drop_tables.c not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd The 1 times to try to start tidb cluster... ***************** properties ***************** "mysql.db"="testSync" "workload"="core" "mysql.host"="127.0.0.1" "mysql.user"="root" "dotransactions"="false" "readallfields"="true" "recordcount"="10" "readproportion"="0" "scanproportion"="0" "mysql.port"="4000" "insertproportion"="0" "threadcount"="2" "operationcount"="0" "updateproportion"="0" "requestdistribution"="uniform" ********************************************** Run finished, takes 4.426871ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3501.2, Avg(us): 806, Min(us): 461, Max(us): 1853, 95th(us): 2000, 99th(us): 2000 check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table drop_tables.c not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table drop_tables.c not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table drop_tables.c exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 19.09 secs (195364816 bytes/sec) [Pipeline] { [Pipeline] cache wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31834374000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57, pid:6962, start at 2024-05-06 13:29:24.45853623 +0800 CST m=+5.479578267 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:24.468 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:24.445 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:24.445 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... check diff failed 6-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:29:26 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31833370001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv, pid:3948, start at 2024-05-06 13:29:23.466234367 +0800 CST m=+5.253076651 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:23.475 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:23.470 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:23.470 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31833370001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv, pid:3948, start at 2024-05-06 13:29:23.466234367 +0800 CST m=+5.253076651 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:23.475 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:23.470 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:23.470 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318334140007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv, pid:4028, start at 2024-05-06 13:29:23.468188245 +0800 CST m=+5.199977375 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:23.474 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:23.461 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:23.461 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318355dc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d, pid:4240, start at 2024-05-06 13:29:25.646161565 +0800 CST m=+5.281737036 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:25.653 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:25.623 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:25.623 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31834374000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57, pid:6962, start at 2024-05-06 13:29:24.45853623 +0800 CST m=+5.479578267 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:24.468 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:24.445 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:24.445 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318344e8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57, pid:7038, start at 2024-05-06 13:29:24.548856473 +0800 CST m=+5.519104857 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:24.556 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:24.538 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:24.538 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5353.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318355dc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d, pid:4240, start at 2024-05-06 13:29:25.646161565 +0800 CST m=+5.281737036 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:25.653 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:25.623 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:25.623 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3183576c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d, pid:4323, start at 2024-05-06 13:29:25.770284159 +0800 CST m=+5.355797517 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:25.778 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:25.773 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:25.773 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api_tls/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api_tls/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/http_api_tls/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_api_tls/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/http_api_tls/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8403.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 1-th time, retry later + set +x + tso='449569978795950081 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449569978795950081 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting TLS PD... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 + set +x + tso='449569979011956737 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449569979011956737 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "requestdistribution"="uniform" "recordcount"="10" "mysql.db"="multi_capture_1" "operationcount"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "readallfields"="true" "workload"="core" "dotransactions"="false" "mysql.user"="root" "threadcount"="2" "readproportion"="0" "updateproportion"="0" "scanproportion"="0" "insertproportion"="0" ********************************************** Run finished, takes 9.757867ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1863.1, Avg(us): 1858, Min(us): 1121, Max(us): 4352, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "recordcount"="20" "readproportion"="0" "mysql.port"="4000" "operationcount"="0" "requestdistribution"="uniform" "mysql.db"="changefeed_error" "insertproportion"="0" "threadcount"="4" "mysql.user"="root" "dotransactions"="false" "scanproportion"="0" "mysql.host"="127.0.0.1" "workload"="core" "readallfields"="true" "updateproportion"="0" ********************************************** Run finished, takes 8.512124ms INSERT - Takes(s): 0.0, Count: 20, OPS: 4445.4, Avg(us): 1560, Min(us): 853, Max(us): 3904, 95th(us): 4000, 99th(us): 4000 [Mon May 6 13:29:31 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.54185420.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ***************** properties ***************** "requestdistribution"="uniform" "mysql.port"="4000" "readproportion"="0" "mysql.host"="127.0.0.1" "insertproportion"="0" "readallfields"="true" "mysql.user"="root" "mysql.db"="multi_capture_2" "scanproportion"="0" "threadcount"="2" "operationcount"="0" "dotransactions"="false" "workload"="core" "recordcount"="10" "updateproportion"="0" ********************************************** Run finished, takes 8.357067ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2078.8, Avg(us): 1597, Min(us): 1043, Max(us): 3502, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "readallfields"="true" "requestdistribution"="uniform" "insertproportion"="0" "dotransactions"="false" "mysql.port"="4000" "recordcount"="10" "updateproportion"="0" "mysql.db"="multi_capture_3" "threadcount"="2" "workload"="core" "readproportion"="0" "mysql.host"="127.0.0.1" "mysql.user"="root" "operationcount"="0" "scanproportion"="0" ********************************************** Run finished, takes 10.35943ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1637.4, Avg(us): 1992, Min(us): 1126, Max(us): 4132, 95th(us): 5000, 99th(us): 5000 check diff failed 2-th time, retry later ***************** properties ***************** "mysql.user"="root" "scanproportion"="0" "dotransactions"="false" "readallfields"="true" "insertproportion"="0" "updateproportion"="0" "mysql.db"="multi_capture_4" "mysql.host"="127.0.0.1" "mysql.port"="4000" "recordcount"="10" "workload"="core" "operationcount"="0" "requestdistribution"="uniform" "readproportion"="0" "threadcount"="2" ********************************************** Run finished, takes 9.349744ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1858.1, Avg(us): 1751, Min(us): 1020, Max(us): 3831, 95th(us): 4000, 99th(us): 4000 [Mon May 6 13:29:32 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.85238525.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting TLS TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3183d6400017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5vlz1-83nq6-835n1, pid:6646, start at 2024-05-06 13:29:33.86234643 +0800 CST m=+5.435386105 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:33.871 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:33.840 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:33.840 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:29:34 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c66231dd-bb4e-43df-ab5a-a835723bc431 {"id":"c66231dd-bb4e-43df-ab5a-a835723bc431","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973371} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a0b2f4 c66231dd-bb4e-43df-ab5a-a835723bc431 /tidb/cdc/default/default/upstream/7365754474880596156 {"id":7365754474880596156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c66231dd-bb4e-43df-ab5a-a835723bc431 {"id":"c66231dd-bb4e-43df-ab5a-a835723bc431","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973371} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a0b2f4 c66231dd-bb4e-43df-ab5a-a835723bc431 /tidb/cdc/default/default/upstream/7365754474880596156 {"id":7365754474880596156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c66231dd-bb4e-43df-ab5a-a835723bc431 {"id":"c66231dd-bb4e-43df-ab5a-a835723bc431","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973371} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a0b2f4 c66231dd-bb4e-43df-ab5a-a835723bc431 /tidb/cdc/default/default/upstream/7365754474880596156 {"id":7365754474880596156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5479.out cli changefeed create --start-ts=449569978795950081 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7365754474880596156,"namespace":"default","id":"changefeed-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T13:29:34.908286435+08:00","start_ts":449569978795950081,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569978795950081,"checkpoint_ts":449569978795950081,"checkpoint_time":"2024-05-06 13:29:28.820"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:29:35 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f28a81f7-d730-457c-8da2-b2cf941decc9 {"id":"f28a81f7-d730-457c-8da2-b2cf941decc9","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973372} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a3b93a f28a81f7-d730-457c-8da2-b2cf941decc9 /tidb/cdc/default/default/upstream/7365754475134638277 {"id":7365754475134638277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f28a81f7-d730-457c-8da2-b2cf941decc9 {"id":"f28a81f7-d730-457c-8da2-b2cf941decc9","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973372} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a3b93a f28a81f7-d730-457c-8da2-b2cf941decc9 /tidb/cdc/default/default/upstream/7365754475134638277 {"id":7365754475134638277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f28a81f7-d730-457c-8da2-b2cf941decc9 {"id":"f28a81f7-d730-457c-8da2-b2cf941decc9","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973372} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a3b93a f28a81f7-d730-457c-8da2-b2cf941decc9 /tidb/cdc/default/default/upstream/7365754475134638277 {"id":7365754475134638277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 13:29:35 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.85788580.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting TLS TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449569978795950081, "checkpoint_time": "2024-05-06 13:29:28.820", "error": { "time": "2024-05-06T13:29:35.018440281+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449569978795949781 is earlier than or equal to GC safepoint at 449569978795950081" } }' + echo '{ "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449569978795950081, "checkpoint_time": "2024-05-06 13:29:28.820", "error": { "time": "2024-05-06T13:29:35.018440281+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449569978795949781 is earlier than or equal to GC safepoint at 449569978795950081" } }' { "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449569978795950081, "checkpoint_time": "2024-05-06 13:29:28.820", "error": { "time": "2024-05-06T13:29:35.018440281+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449569978795949781 is earlier than or equal to GC safepoint at 449569978795950081" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365754474880596156, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449569978795950081, '"checkpoint_time":' '"2024-05-06' '13:29:28.820",' '"error":' '{' '"time":' '"2024-05-06T13:29:35.018440281+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449569978795949781 is earlier than or equal to GC safepoint at '449569978795950081"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365754474880596156, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449569978795950081, '"checkpoint_time":' '"2024-05-06' '13:29:28.820",' '"error":' '{' '"time":' '"2024-05-06T13:29:35.018440281+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449569978795949781 is earlier than or equal to GC safepoint at '449569978795950081"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449569978795949781 is earlier than or equal to GC safepoint at 449569978795950081' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449569978795949781 is earlier than or equal to GC safepoint at 449569978795950081 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5557.out cli changefeed resume -c changefeed-error VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3183d6400017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5vlz1-83nq6-835n1, pid:6646, start at 2024-05-06 13:29:33.86234643 +0800 CST m=+5.435386105 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:33.871 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:33.840 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:33.840 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3183d7cc0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5vlz1-83nq6-835n1, pid:6711, start at 2024-05-06 13:29:33.976316923 +0800 CST m=+5.467568421 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:33.983 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:33.988 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:33.988 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_source/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_source/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 10.81 secs (344982048 bytes/sec) [Pipeline] { [Pipeline] cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:29:38 GMT < Content-Length: 1259 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/35afa740-f19f-4c03-9568-99a6c40885d4 {"id":"35afa740-f19f-4c03-9568-99a6c40885d4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973375} /tidb/cdc/default/__cdc_meta__/capture/f28a81f7-d730-457c-8da2-b2cf941decc9 {"id":"f28a81f7-d730-457c-8da2-b2cf941decc9","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973372} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a3b93a f28a81f7-d730-457c-8da2-b2cf941decc9 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a3b96d 35afa740-f19f-4c03-9568-99a6c40885d4 /tidb/cdc/default/default/upstream/7365754475134638277 {"id":7365754475134638277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/35afa740-f19f-4c03-9568-99a6c40885d4 {"id":"35afa740-f19f-4c03-9568-99a6c40885d4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973375} /tidb/cdc/default/__cdc_meta__/capture/f28a81f7-d730-457c-8da2-b2cf941decc9 {"id":"f28a81f7-d730-457c-8da2-b2cf941decc9","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973372} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a3b93a f28a81f7-d730-457c-8da2-b2cf941decc9 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a3b96d 35afa740-f19f-4c03-9568-99a6c40885d4 /tidb/cdc/default/default/upstream/7365754475134638277 {"id":7365754475134638277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/35afa740-f19f-4c03-9568-99a6c40885d4 {"id":"35afa740-f19f-4c03-9568-99a6c40885d4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973375} /tidb/cdc/default/__cdc_meta__/capture/f28a81f7-d730-457c-8da2-b2cf941decc9 {"id":"f28a81f7-d730-457c-8da2-b2cf941decc9","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973372} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a3b93a f28a81f7-d730-457c-8da2-b2cf941decc9 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a3b96d 35afa740-f19f-4c03-9568-99a6c40885d4 /tidb/cdc/default/default/upstream/7365754475134638277 {"id":7365754475134638277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 13:29:38 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.86288630.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x table changefeed_error.usertable exists check diff failed 1-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8017.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 6-th time, retry later + set +x + tso='449569981501276161 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449569981501276161 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:29:40 CST 2024] <<<<<< START cdc server in multi_source case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.80428044.out server --log-file /tmp/tidb_cdc_test/multi_source/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_source/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff successfully ***************** properties ***************** "readallfields"="true" "workload"="core" "scanproportion"="0" "updateproportion"="0" "requestdistribution"="uniform" "readproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "dotransactions"="false" "insertproportion"="0" "mysql.db"="changefeed_error" "recordcount"="20" "mysql.user"="root" "operationcount"="0" "threadcount"="4" ********************************************** Run finished, takes 4.404246ms INSERT - Takes(s): 0.0, Count: 20, OPS: 7565.5, Avg(us): 757, Min(us): 417, Max(us): 2234, 95th(us): 2000, 99th(us): 3000 check diff successfully {"id":"c66231dd-bb4e-43df-ab5a-a835723bc431","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973371} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/c66231dd-bb4e-43df-ab5a-a835723bc431 + [[ /tidb/cdc/default/__cdc_meta__/capture/c66231dd-bb4e-43df-ab5a-a835723bc431 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/c66231dd-bb4e-43df-ab5a-a835723bc431' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/c66231dd-bb4e-43df-ab5a-a835723bc431 + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:29:41 GMT < Content-Length: 1732 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/35afa740-f19f-4c03-9568-99a6c40885d4 {"id":"35afa740-f19f-4c03-9568-99a6c40885d4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973375} /tidb/cdc/default/__cdc_meta__/capture/8c28fcdc-1a35-468e-a545-6c5f2e17f0a1 {"id":"8c28fcdc-1a35-468e-a545-6c5f2e17f0a1","address":"127.0.0.1:8303","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973378} /tidb/cdc/default/__cdc_meta__/capture/f28a81f7-d730-457c-8da2-b2cf941decc9 {"id":"f28a81f7-d730-457c-8da2-b2cf941decc9","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973372} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a3b93a f28a81f7-d730-457c-8da2-b2cf941decc9 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a3b96d 35afa740-f19f-4c03-9568-99a6c40885d4 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a3b98d 8c28fcdc-1a35-468e-a545-6c5f2e17f0a1 /tidb/cdc/default/default/upstream/7365754475134638277 {"id":7365754475134638277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/35afa740-f19f-4c03-9568-99a6c40885d4 {"id":"35afa740-f19f-4c03-9568-99a6c40885d4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973375} /tidb/cdc/default/__cdc_meta__/capture/8c28fcdc-1a35-468e-a545-6c5f2e17f0a1 {"id":"8c28fcdc-1a35-468e-a545-6c5f2e17f0a1","address":"127.0.0.1:8303","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973378} /tidb/cdc/default/__cdc_meta__/capture/f28a81f7-d730-457c-8da2-b2cf941decc9 {"id":"f28a81f7-d730-457c-8da2-b2cf941decc9","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973372} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a3b93a f28a81f7-d730-457c-8da2-b2cf941decc9 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a3b96d 35afa740-f19f-4c03-9568-99a6c40885d4 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a3b98d 8c28fcdc-1a35-468e-a545-6c5f2e17f0a1 /tidb/cdc/default/default/upstream/7365754475134638277 {"id":7365754475134638277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/35afa740-f19f-4c03-9568-99a6c40885d4 {"id":"35afa740-f19f-4c03-9568-99a6c40885d4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973375} /tidb/cdc/default/__cdc_meta__/capture/8c28fcdc-1a35-468e-a545-6c5f2e17f0a1 {"id":"8c28fcdc-1a35-468e-a545-6c5f2e17f0a1","address":"127.0.0.1:8303","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973378} /tidb/cdc/default/__cdc_meta__/capture/f28a81f7-d730-457c-8da2-b2cf941decc9 {"id":"f28a81f7-d730-457c-8da2-b2cf941decc9","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973372} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a3b93a f28a81f7-d730-457c-8da2-b2cf941decc9 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a3b96d 35afa740-f19f-4c03-9568-99a6c40885d4 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a3b98d 8c28fcdc-1a35-468e-a545-6c5f2e17f0a1 /tidb/cdc/default/default/upstream/7365754475134638277 {"id":7365754475134638277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8688.out cli changefeed create --start-ts=449569979011956737 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 Create changefeed successfully! ID: b6b4ea80-0e2c-4170-a0de-a20d094fa8b4 Info: {"upstream_id":7365754475134638277,"namespace":"default","id":"b6b4ea80-0e2c-4170-a0de-a20d094fa8b4","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:29:41.66103377+08:00","start_ts":449569979011956737,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569979011956737,"checkpoint_ts":449569979011956737,"checkpoint_time":"2024-05-06 13:29:29.644"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31844a980014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d, pid:5930, start at 2024-05-06 13:29:41.313183984 +0800 CST m=+5.223536627 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:41.320 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:41.286 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:41.286 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Mon May 6 13:29:42 CST 2024] <<<<<< START cdc server in http_api_tls case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls.59935995.out server --log-file /tmp/tidb_cdc_test/http_api_tls/cdc_http_api_tls_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls/cdc_data_http_api_tls_tls1 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x table multi_capture_1.usertable not exists for 1-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/c66231dd-bb4e-43df-ab5a-a835723bc431 + [[ /tidb/cdc/default/__cdc_meta__/capture/c66231dd-bb4e-43df-ab5a-a835723bc431 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/c66231dd-bb4e-43df-ab5a-a835723bc431' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/c66231dd-bb4e-43df-ab5a-a835723bc431 + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:29:43 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18f3e7b4-5e40-436d-acfc-b80839bcf863 {"id":"18f3e7b4-5e40-436d-acfc-b80839bcf863","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973380} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60c86bd9 18f3e7b4-5e40-436d-acfc-b80839bcf863 /tidb/cdc/default/default/upstream/7365754510994266001 {"id":7365754510994266001,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18f3e7b4-5e40-436d-acfc-b80839bcf863 {"id":"18f3e7b4-5e40-436d-acfc-b80839bcf863","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973380} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60c86bd9 18f3e7b4-5e40-436d-acfc-b80839bcf863 /tidb/cdc/default/default/upstream/7365754510994266001 {"id":7365754510994266001,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18f3e7b4-5e40-436d-acfc-b80839bcf863 {"id":"18f3e7b4-5e40-436d-acfc-b80839bcf863","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973380} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60c86bd9 18f3e7b4-5e40-436d-acfc-b80839bcf863 /tidb/cdc/default/default/upstream/7365754510994266001 {"id":7365754510994266001,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8095.out cli changefeed create --start-ts=449569981501276161 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 79d4b137-48f9-4fa8-b3ac-ac3212f16af3 Info: {"upstream_id":7365754510994266001,"namespace":"default","id":"79d4b137-48f9-4fa8-b3ac-ac3212f16af3","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:29:44.130021561+08:00","start_ts":449569981501276161,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569981501276161,"checkpoint_ts":449569981501276161,"checkpoint_time":"2024-05-06 13:29:39.140"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table multi_capture_1.usertable exists table multi_capture_2.usertable not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + set +x go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:29:45 GMT < Content-Length: 1209 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7b03f11-123c-489c-b27d-daea1f81d987 {"id":"b7b03f11-123c-489c-b27d-daea1f81d987","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973382} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c60e99666 b7b03f11-123c-489c-b27d-daea1f81d987 /tidb/cdc/default/default/upstream/7365754547335523651 {"id":7365754547335523651,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7b03f11-123c-489c-b27d-daea1f81d987 {"id":"b7b03f11-123c-489c-b27d-daea1f81d987","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973382} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c60e99666 b7b03f11-123c-489c-b27d-daea1f81d987 /tidb/cdc/default/default/upstream/7365754547335523651 {"id":7365754547335523651,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7b03f11-123c-489c-b27d-daea1f81d987 {"id":"b7b03f11-123c-489c-b27d-daea1f81d987","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973382} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c60e99666 b7b03f11-123c-489c-b27d-daea1f81d987 /tidb/cdc/default/default/upstream/7365754547335523651 {"id":7365754547335523651,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda check diff failed 2-th time, retry later go: downloading golang.org/x/text v0.14.0 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 6.42 secs (580725684 bytes/sec) [Pipeline] { [Pipeline] cache check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/c66231dd-bb4e-43df-ab5a-a835723bc431 + [[ /tidb/cdc/default/__cdc_meta__/capture/c66231dd-bb4e-43df-ab5a-a835723bc431 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/c66231dd-bb4e-43df-ab5a-a835723bc431' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/c66231dd-bb4e-43df-ab5a-a835723bc431 + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later [Mon May 6 13:29:47 CST 2024] <<<<<< START cdc server in http_api_tls case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls.60516053.out server --log-file /tmp/tidb_cdc_test/http_api_tls/cdc_http_api_tls_tls2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls/cdc_data_http_api_tls_tls2 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8301 --pd https://127.0.0.1:2579 ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later check diff failed 3-th time, retry later go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/stretchr/testify v1.9.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading cloud.google.com/go v0.112.2 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/tools v0.20.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading cloud.google.com/go/compute v1.25.1 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/kr/pretty v0.3.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 Verifying downstream PD is started... go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading go.opentelemetry.io/otel/metric v1.24.0 table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 4-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:29:50 GMT < Content-Length: 1659 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7b03f11-123c-489c-b27d-daea1f81d987 {"id":"b7b03f11-123c-489c-b27d-daea1f81d987","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973382} /tidb/cdc/default/__cdc_meta__/capture/cfed0bc4-d96f-45dd-8b08-67b27c27511b {"id":"cfed0bc4-d96f-45dd-8b08-67b27c27511b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973387} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c60e99666 b7b03f11-123c-489c-b27d-daea1f81d987 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c60e996a5 cfed0bc4-d96f-45dd-8b08-67b27c27511b /tidb/cdc/default/default/upstream/7365754547335523651 {"id":7365754547335523651,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7b03f11-123c-489c-b27d-daea1f81d987 {"id":"b7b03f11-123c-489c-b27d-daea1f81d987","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973382} /tidb/cdc/default/__cdc_meta__/capture/cfed0bc4-d96f-45dd-8b08-67b27c27511b {"id":"cfed0bc4-d96f-45dd-8b08-67b27c27511b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973387} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c60e99666 b7b03f11-123c-489c-b27d-daea1f81d987 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c60e996a5 cfed0bc4-d96f-45dd-8b08-67b27c27511b /tidb/cdc/default/default/upstream/7365754547335523651 {"id":7365754547335523651,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7b03f11-123c-489c-b27d-daea1f81d987 {"id":"b7b03f11-123c-489c-b27d-daea1f81d987","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973382} /tidb/cdc/default/__cdc_meta__/capture/cfed0bc4-d96f-45dd-8b08-67b27c27511b {"id":"cfed0bc4-d96f-45dd-8b08-67b27c27511b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973387} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c60e99666 b7b03f11-123c-489c-b27d-daea1f81d987 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c60e996a5 cfed0bc4-d96f-45dd-8b08-67b27c27511b /tidb/cdc/default/default/upstream/7365754547335523651 {"id":7365754547335523651,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x table multi_capture_4.usertable exists check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later pass test: check health go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 pass test: get status go: downloading github.com/jmespath/go-jmespath v0.4.0 check diff successfully ***************** properties ***************** "recordcount"="20" "readallfields"="true" "mysql.host"="127.0.0.1" "insertproportion"="0" "workload"="core" "mysql.user"="root" "readproportion"="0" "mysql.db"="multi_capture_1" "requestdistribution"="uniform" "operationcount"="0" "dotransactions"="false" "updateproportion"="0" "scanproportion"="0" "threadcount"="2" "mysql.port"="4000" ********************************************** Run finished, takes 11.009777ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2026.1, Avg(us): 1059, Min(us): 501, Max(us): 1817, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "scanproportion"="0" "workload"="core" "readproportion"="0" "mysql.host"="127.0.0.1" "updateproportion"="0" "insertproportion"="0" "mysql.port"="4000" "readallfields"="true" "mysql.user"="root" "threadcount"="2" "recordcount"="20" "operationcount"="0" "requestdistribution"="uniform" "mysql.db"="multi_capture_2" "dotransactions"="false" ********************************************** Run finished, takes 11.043266ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2031.8, Avg(us): 1038, Min(us): 513, Max(us): 1753, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "updateproportion"="0" "threadcount"="2" "insertproportion"="0" "recordcount"="20" "mysql.user"="root" "scanproportion"="0" "requestdistribution"="uniform" "readallfields"="true" "dotransactions"="false" "mysql.db"="multi_capture_3" "readproportion"="0" "workload"="core" "mysql.port"="4000" "operationcount"="0" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 12.537753ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1789.5, Avg(us): 1212, Min(us): 557, Max(us): 2088, 95th(us): 2000, 99th(us): 3000 ***************** properties ***************** "threadcount"="2" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "readallfields"="true" "mysql.db"="multi_capture_4" "updateproportion"="0" "mysql.user"="root" "scanproportion"="0" "recordcount"="20" "dotransactions"="false" "workload"="core" "readproportion"="0" "operationcount"="0" "mysql.port"="4000" "insertproportion"="0" ********************************************** Run finished, takes 11.173369ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2019.4, Avg(us): 1061, Min(us): 447, Max(us): 1772, 95th(us): 2000, 99th(us): 2000 check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 13:29:53 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.58105812.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd pass test: create changefeed check_changefeed_state https://127.0.0.1:2579 changefeed-test1 normal null /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test1 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test1 -s + info='{ "upstream_id": 7365754547335523651, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 449569985183612929, "checkpoint_time": "2024-05-06 13:29:53.187", "error": null }' + echo '{ "upstream_id": 7365754547335523651, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 449569985183612929, "checkpoint_time": "2024-05-06 13:29:53.187", "error": null }' { "upstream_id": 7365754547335523651, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 449569985183612929, "checkpoint_time": "2024-05-06 13:29:53.187", "error": null } ++ echo '{' '"upstream_id":' 7365754547335523651, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 449569985183612929, '"checkpoint_time":' '"2024-05-06' '13:29:53.187",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365754547335523651, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 449569985183612929, '"checkpoint_time":' '"2024-05-06' '13:29:53.187",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test2 normal null /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test2 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test2 -s check diff successfully + info='{ "upstream_id": 7365754547335523651, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 449569985248886785, "checkpoint_time": "2024-05-06 13:29:53.436", "error": null }' + echo '{ "upstream_id": 7365754547335523651, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 449569985248886785, "checkpoint_time": "2024-05-06 13:29:53.436", "error": null }' { "upstream_id": 7365754547335523651, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 449569985248886785, "checkpoint_time": "2024-05-06 13:29:53.436", "error": null } ++ echo '{' '"upstream_id":' 7365754547335523651, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 449569985248886785, '"checkpoint_time":' '"2024-05-06' '13:29:53.436",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365754547335523651, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 449569985248886785, '"checkpoint_time":' '"2024-05-06' '13:29:53.436",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test3 normal null /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test3 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test3 -s + info='{ "upstream_id": 7365754547335523651, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 449569985301577729, "checkpoint_time": "2024-05-06 13:29:53.637", "error": null }' + echo '{ "upstream_id": 7365754547335523651, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 449569985301577729, "checkpoint_time": "2024-05-06 13:29:53.637", "error": null }' { "upstream_id": 7365754547335523651, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 449569985301577729, "checkpoint_time": "2024-05-06 13:29:53.637", "error": null } ++ echo '{' '"upstream_id":' 7365754547335523651, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 449569985301577729, '"checkpoint_time":' '"2024-05-06' '13:29:53.637",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365754547335523651, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 449569985301577729, '"checkpoint_time":' '"2024-05-06' '13:29:53.637",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test4 normal null /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test4 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test4 -s + info='{ "upstream_id": 7365754547335523651, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 449569985353744385, "checkpoint_time": "2024-05-06 13:29:53.836", "error": null }' + echo '{ "upstream_id": 7365754547335523651, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 449569985353744385, "checkpoint_time": "2024-05-06 13:29:53.836", "error": null }' { "upstream_id": 7365754547335523651, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 449569985353744385, "checkpoint_time": "2024-05-06 13:29:53.836", "error": null } ++ echo '{' '"upstream_id":' 7365754547335523651, '"namespace":' '"default",' '"id":' '"changefeed-test4",' '"state":' '"normal",' '"checkpoint_tso":' 449569985353744385, '"checkpoint_time":' '"2024-05-06' '13:29:53.836",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365754547335523651, '"namespace":' '"default",' '"id":' '"changefeed-test4",' '"state":' '"normal",' '"checkpoint_tso":' 449569985353744385, '"checkpoint_time":' '"2024-05-06' '13:29:53.836",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:29:56 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ff814f22-1e6e-425d-8c5d-86442bcfc249 {"id":"ff814f22-1e6e-425d-8c5d-86442bcfc249","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973393} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a0b3ce ff814f22-1e6e-425d-8c5d-86442bcfc249 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365754474880596156,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:29:34.908286435+08:00","start-ts":449569978795950081,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-06T13:29:53.675478981+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569985310228481} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449569981797498890,"min-table-barrier-ts":449569981797498890,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365754474880596156 {"id":7365754474880596156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ff814f22-1e6e-425d-8c5d-86442bcfc249 {"id":"ff814f22-1e6e-425d-8c5d-86442bcfc249","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973393} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a0b3ce ff814f22-1e6e-425d-8c5d-86442bcfc249 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365754474880596156,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:29:34.908286435+08:00","start-ts":449569978795950081,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-06T13:29:53.675478981+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569985310228481} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449569981797498890,"min-table-barrier-ts":449569981797498890,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365754474880596156 + grep -q 'failed to get info:' {"id":7365754474880596156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ff814f22-1e6e-425d-8c5d-86442bcfc249 {"id":"ff814f22-1e6e-425d-8c5d-86442bcfc249","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973393} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a0b3ce ff814f22-1e6e-425d-8c5d-86442bcfc249 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365754474880596156,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:29:34.908286435+08:00","start-ts":449569978795950081,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-06T13:29:53.675478981+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569985310228481} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449569981797498890,"min-table-barrier-ts":449569981797498890,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365754474880596156 {"id":7365754474880596156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449569981797498890, "checkpoint_time": "2024-05-06 13:29:40.270", "error": { "time": "2024-05-06T13:29:53.675478981+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449569981797498890, "checkpoint_time": "2024-05-06 13:29:40.270", "error": { "time": "2024-05-06T13:29:53.675478981+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449569981797498890, "checkpoint_time": "2024-05-06 13:29:40.270", "error": { "time": "2024-05-06T13:29:53.675478981+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7365754474880596156, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449569981797498890, '"checkpoint_time":' '"2024-05-06' '13:29:40.270",' '"error":' '{' '"time":' '"2024-05-06T13:29:53.675478981+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7365754474880596156, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449569981797498890, '"checkpoint_time":' '"2024-05-06' '13:29:40.270",' '"error":' '{' '"time":' '"2024-05-06T13:29:53.675478981+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5913.out cli changefeed remove -c changefeed-error wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... Changefeed remove successfully. ID: changefeed-error CheckpointTs: 449569981797498890 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318538440006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3, pid:6347, start at 2024-05-06 13:29:56.502330385 +0800 CST m=+5.065286291 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:56.508 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:56.497 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:56.497 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318538440006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3, pid:6347, start at 2024-05-06 13:29:56.502330385 +0800 CST m=+5.065286291 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:56.508 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:56.497 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:56.497 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31853c5c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3, pid:6433, start at 2024-05-06 13:29:56.786460669 +0800 CST m=+5.261998129 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:31:56.793 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:29:56.759 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:19:56.759 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:29:57 CST 2024] <<<<<< run test case multi_capture success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 9.13 secs (408505419 bytes/sec) [Pipeline] { [Pipeline] cache table test.simple1 exists pass test: list changefeed + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully check diff failed 2-th time, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... pass test: get changefeed [2024/05/06 13:29:50.792 +08:00] [INFO] [main.go:99] ["running ddl test: 1 modifyColumnDefaultValueDDL2"] [2024/05/06 13:29:50.792 +08:00] [INFO] [main.go:99] ["running ddl test: 0 modifyColumnDefaultValueDDL1"] [2024/05/06 13:29:51.319 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb887bef5_9989_41ce_8707_241a292686ea"] [2024/05/06 13:29:51.323 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb991592e_aa4d_4f42_be4c_28c82db19568"] [2024/05/06 13:29:51.324 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa924f5cd_4576_464f_a68b_59926d715d2d"] [2024/05/06 13:29:51.324 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs7e8b3bbc_5085_4b7f_8060_f3dad54b77f0"] [2024/05/06 13:29:51.325 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb0616ec8_a21a_4d39_bb43_5a1d133c8396"] [2024/05/06 13:29:51.326 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLse4b265cb_7d12_4584_999d_ed584fa0b4d5"] [2024/05/06 13:29:51.329 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsbc7020c9_287d_4136_a8ed_fd460a9178f6"] [2024/05/06 13:29:51.330 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs2c23c707_1200_4cc5_92d5_391fd26007bf"] [2024/05/06 13:29:51.547 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/06 13:29:51.547 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/06 13:29:51.822 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/06 13:29:51.823 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/06 13:29:52.323 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:29:52.325 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:29:52.328 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:29:52.330 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:29:52.330 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:29:52.334 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:29:52.336 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:29:52.343 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:29:52.445 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:29:52.446 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:29:52.449 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:29:52.519 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:29:52.519 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:29:52.520 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:29:52.522 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:29:52.523 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:29:52.642 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/06 13:29:52.645 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/06 13:29:53.021 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/06 13:29:53.026 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/06 13:29:53.033 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/06 13:29:53.038 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/06 13:29:53.233 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:29:53.235 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:29:53.239 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:29:53.240 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:29:53.241 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:29:53.241 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:29:53.243 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:29:53.330 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:29:53.525 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:29:53.526 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:29:53.528 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:29:53.529 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:29:53.531 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:29:53.533 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:29:53.535 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:29:53.543 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:29:53.634 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/06 13:29:53.642 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/06 13:29:54.336 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:29:54.420 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:29:54.421 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:29:54.423 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:29:54.425 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:29:54.426 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:29:54.436 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:29:54.439 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:29:54.442 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:29:54.442 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:29:54.444 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:29:54.521 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:29:54.522 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:29:54.524 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:29:54.524 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:29:54.525 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:29:54.533 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/06 13:29:54.536 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/06 13:29:54.553 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/06 13:29:54.620 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/06 13:29:55.328 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:29:55.338 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:29:55.343 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:29:55.419 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:29:55.419 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:29:55.421 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:29:55.429 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:29:55.430 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:29:55.430 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:29:55.430 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:29:55.436 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:29:55.439 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:29:55.439 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:29:55.441 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:29:55.442 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:29:55.521 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:29:55.533 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/06 13:29:55.536 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/06 13:29:55.841 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/06 13:29:55.919 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/06 13:29:55.929 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/06 13:29:55.934 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/06 13:29:56.247 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:29:56.331 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:29:56.338 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:29:56.420 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:29:56.426 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:29:56.426 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:29:56.430 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:29:56.431 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:29:56.434 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:29:56.434 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:29:56.435 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:29:56.437 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:29:56.439 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:29:56.441 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:29:56.443 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:29:56.531 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/06 13:29:56.531 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/06 13:29:56.534 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:29:57.322 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/06 13:29:57.334 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:29:57.338 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/06 13:29:57.346 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:29:57.420 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:29:57.434 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:29:57.437 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:29:57.443 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:29:57.443 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:29:57.445 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:29:57.445 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:29:57.526 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:29:57.530 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:29:57.537 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:29:57.538 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:29:57.541 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:29:57.546 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:29:57.551 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:29:57.632 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/05/06 13:29:57.633 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/05/06 13:29:58.333 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:29:58.334 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:29:58.432 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:29:58.521 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:29:58.523 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:29:58.525 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:29:58.533 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:29:58.538 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:29:58.539 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:29:58.539 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:29:58.543 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:29:58.621 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:29:58.627 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:29:58.628 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:29:58.632 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:29:58.632 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:29:58.639 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/05/06 13:29:58.642 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/05/06 13:29:58.645 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/06 13:29:58.649 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/06 13:29:58.722 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/06 13:29:58.727 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/06 13:29:59.328 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:29:59.340 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:29:59.519 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:29:59.530 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:29:59.538 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:29:59.539 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:29:59.539 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:29:59.620 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:29:59.626 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:29:59.630 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:29:59.633 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:29:59.634 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:29:59.645 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:29:59.646 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:29:59.646 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:29:59.718 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/06 13:29:59.726 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:29:59.730 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/06 13:30:00.039 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/05/06 13:30:00.126 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [Mon May 6 13:29:59 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.77927794.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 13:30:00.337 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:00.421 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:00.531 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:00.535 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:00.540 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 13:30:00 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.60356037.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later [2024/05/06 13:30:00.633 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:00.634 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:00.646 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:00.723 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:00.731 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:00.736 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/05/06 13:30:00.740 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:00.745 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/05/06 13:30:00.820 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:00.821 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:00.826 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:00.827 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:00.828 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] pass test: pause changefeed [2024/05/06 13:30:01.423 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:01.520 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:01.540 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/06 13:30:01.618 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/05/06 13:30:01.618 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:01.619 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:01.626 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/06 13:30:01.631 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/05/06 13:30:01.639 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:01.720 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:01.726 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:01.732 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:01.732 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:01.736 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/05/06 13:30:01.823 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/05/06 13:30:01.823 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:01.824 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:01.831 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:01.834 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:01.835 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:01.838 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:01.841 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:02.337 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] check diff failed 4-th time, retry later [2024/05/06 13:30:02.538 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:02.540 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:02.644 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > pass test: update changefeed [2024/05/06 13:30:02.721 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:02.732 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:02.739 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:02.824 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/06 13:30:02.923 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/06 13:30:02.925 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:02.925 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:02.932 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:02.933 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:02.939 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:02 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e632786-0788-4538-9d0d-4bc2e5e9a05b {"id":"3e632786-0788-4538-9d0d-4bc2e5e9a05b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973400} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61224dc9 3e632786-0788-4538-9d0d-4bc2e5e9a05b /tidb/cdc/default/default/upstream/7365754613272788698 {"id":7365754613272788698,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e632786-0788-4538-9d0d-4bc2e5e9a05b {"id":"3e632786-0788-4538-9d0d-4bc2e5e9a05b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973400} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61224dc9 3e632786-0788-4538-9d0d-4bc2e5e9a05b /tidb/cdc/default/default/upstream/7365754613272788698 {"id":7365754613272788698,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e632786-0788-4538-9d0d-4bc2e5e9a05b {"id":"3e632786-0788-4538-9d0d-4bc2e5e9a05b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973400} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61224dc9 3e632786-0788-4538-9d0d-4bc2e5e9a05b /tidb/cdc/default/default/upstream/7365754613272788698 {"id":7365754613272788698,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/06 13:30:03.022 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:03.031 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:03.039 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:03.043 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:03.123 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/05/06 13:30:03.137 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] table processor_stop_delay.t not exists for 1-th check, retry later [2024/05/06 13:30:03.521 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:03.635 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:03.733 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:03 GMT < Content-Length: 877 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7c39171-8890-449c-904d-7b7dd0a0238d {"id":"c7c39171-8890-449c-904d-7b7dd0a0238d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973400} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a0b42c c7c39171-8890-449c-904d-7b7dd0a0238d /tidb/cdc/default/default/upstream/7365754474880596156 {"id":7365754474880596156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7c39171-8890-449c-904d-7b7dd0a0238d {"id":"c7c39171-8890-449c-904d-7b7dd0a0238d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973400} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a0b42c c7c39171-8890-449c-904d-7b7dd0a0238d /tidb/cdc/default/default/upstream/7365754474880596156 {"id":7365754474880596156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7c39171-8890-449c-904d-7b7dd0a0238d {"id":"c7c39171-8890-449c-904d-7b7dd0a0238d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973400} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a0b42c c7c39171-8890-449c-904d-7b7dd0a0238d /tidb/cdc/default/default/upstream/7365754474880596156 {"id":7365754474880596156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6083.out cli changefeed create --start-ts=449569978795950081 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-1 Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7365754474880596156,"namespace":"default","id":"changefeed-error-1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T13:30:03.856418622+08:00","start_ts":449569978795950081,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569978795950081,"checkpoint_ts":449569978795950081,"checkpoint_time":"2024-05-06 13:29:28.820"} PASS [2024/05/06 13:30:03.825 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:03.831 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:03.839 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:03.920 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/06 13:30:03.922 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:03.944 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:04.019 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/06 13:30:04.029 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/06 13:30:04.041 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:04.042 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:04.048 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:04.135 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:04.140 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:04.221 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:04.225 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] pass test: resume changefeed [2024/05/06 13:30:04.628 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 13:30:04.637 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/05/06 13:30:04.641 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [2024/05/06 13:30:04.726 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/05/06 13:30:04.730 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [2024/05/06 13:30:04.818 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 13:30:04.824 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 4-th check, retry later [2024/05/06 13:30:04.923 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 13:30:04.932 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 13:30:04.936 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 13:30:05.021 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/05/06 13:30:05.027 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 13:30:05.038 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 13:30:05.041 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] check diff failed 5-th time, retry later [2024/05/06 13:30:05.134 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 13:30:05.217 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 13:30:05.227 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 13:30:05.334 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 13:30:05.335 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 13:30:05.343 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] + set +x [2024/05/06 13:30:05.628 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] table processor_stop_delay.t exists check diff failed 1-th time, retry later [2024/05/06 13:30:05.732 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 13:30:05.833 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 253 100 253 0 0 1829 0 --:--:-- --:--:-- --:--:-- 1833 + info='{"state":"warning","resolved_ts":449569979189166099,"checkpoint_ts":449569979189166099,"last_warning":{"time":"2024-05-06T13:30:05.537935071+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":449569979189166099,"checkpoint_ts":449569979189166099,"last_warning":{"time":"2024-05-06T13:30:05.537935071+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":449569979189166099,"checkpoint_ts":449569979189166099,"last_warning":{"time":"2024-05-06T13:30:05.537935071+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ jq -r .state ++ echo '{"state":"warning","resolved_ts":449569979189166099,"checkpoint_ts":449569979189166099,"last_warning":{"time":"2024-05-06T13:30:05.537935071+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ jq -r .last_warning.message ++ echo '{"state":"warning","resolved_ts":449569979189166099,"checkpoint_ts":449569979189166099,"last_warning":{"time":"2024-05-06T13:30:05.537935071+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6144.out cli changefeed remove -c changefeed-error-1 [2024/05/06 13:30:05.919 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 13:30:05.933 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] [2024/05/06 13:30:05.937 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 13:30:05.947 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] [2024/05/06 13:30:06.037 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 13:30:06.117 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/05/06 13:30:06.124 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 13:30:06.132 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/05/06 13:30:06.135 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] pass test: rebalance table [2024/05/06 13:30:06.228 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 13:30:06.236 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 13:30:06.346 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 13:30:06.524 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] table test.finish_mark not exists for 5-th check, retry later [2024/05/06 13:30:06.726 +08:00] [INFO] [main.go:178] ["1 insert success: 1600"] [2024/05/06 13:30:06.738 +08:00] [INFO] [main.go:178] ["1 insert success: 1600"] [2024/05/06 13:30:06.756 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 449569979189166099 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff successfully pass test: list processors [2024/05/06 13:30:06.928 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 13:30:06.948 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 13:30:07.019 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 13:30:07.047 +08:00] [INFO] [main.go:178] ["0 insert success: 1200"] [2024/05/06 13:30:07.050 +08:00] [INFO] [main.go:199] ["0 delete success: 600"] [2024/05/06 13:30:07.059 +08:00] [INFO] [main.go:178] ["0 insert success: 1200"] [2024/05/06 13:30:07.063 +08:00] [INFO] [main.go:199] ["0 delete success: 600"] [2024/05/06 13:30:07.160 +08:00] [INFO] [main.go:178] ["1 insert success: 1700"] [2024/05/06 13:30:07.222 +08:00] [INFO] [main.go:178] ["1 insert success: 1700"] [2024/05/06 13:30:07.245 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 13:30:07.431 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 13:30:07.452 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 13:30:07.527 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 13:30:07.630 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsfcf495af_842c_4556_b664_39a85a33d957"] [2024/05/06 13:30:07.634 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd4148b0a_87a7_45b4_bd22_c11dafa00da8"] [2024/05/06 13:30:07.635 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLse3831681_1af4_4161_9420_3e140aa310fa"] [2024/05/06 13:30:07.666 +08:00] [INFO] [main.go:178] ["1 insert success: 1800"] check diff successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status_with_redo + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo The 1 times to try to start tidb cluster... [2024/05/06 13:30:07.757 +08:00] [INFO] [main.go:178] ["0 insert success: 1300"] [2024/05/06 13:30:08.054 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:08.057 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:08.060 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:08.119 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs95e5414a_26e0_406c_affa_ea52bb2fdebf"] [2024/05/06 13:30:08.124 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs7f0a53a9_273c_489a_9524_f8d479f0bd16"] [2024/05/06 13:30:08.131 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs97e7da45_5cfa_447c_9779_b651de8ac4ff"] [2024/05/06 13:30:08.136 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:08.136 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:08.142 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd30ccdad_80e7_450c_8a28_9bdae3e5a0d4"] [2024/05/06 13:30:08.143 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb0a1d7ff_3644_4f76_b70f_dffc1114bf3a"] [2024/05/06 13:30:08.175 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] + set +x table test.finish_mark not exists for 6-th check, retry later check diff failed 1-th time, retry later wait process cdc.test exit for 1-th time... [2024/05/06 13:30:08.918 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:08.923 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:08.927 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:08.948 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:08.948 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:09.018 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:09.027 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:09.028 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:09.030 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:09.033 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:09.034 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:09.041 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:09.042 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:09.042 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:09.042 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:09.051 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] wait process cdc.test exit for 2-th time... check diff failed 1-th time, retry later [2024/05/06 13:30:09.539 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:09.619 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:09.620 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:09.620 +08:00] [INFO] [main.go:88] ["testGetDefaultValue take 18.827619984s"] [2024/05/06 13:30:09.740 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:09.744 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:09.819 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:09.822 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:09.829 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:09.830 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:09.831 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:09.831 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:09.832 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:09.835 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:09.835 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:09.836 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:09.839 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 13:30:09 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.62006202.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 13:30:10.142 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:10.237 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:10.242 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:10.423 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] table test.finish_mark not exists for 7-th check, retry later [2024/05/06 13:30:10.544 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:10.545 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:10.623 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:10.625 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:10.628 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:10.629 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:10.629 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:10.633 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:10.635 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:10.635 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:10.636 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:10.639 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] check diff failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/06 13:30:11.027 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:11.226 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] pass test: get processor check diff failed 2-th time, retry later [2024/05/06 13:30:11.329 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:11.825 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] pass test: move table [2024/05/06 13:30:12.228 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:12.320 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:12.327 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] table test.finish_mark not exists for 8-th check, retry later [2024/05/06 13:30:12.418 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:12.424 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:12.425 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:30:12.425 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:12.426 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:12.426 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:12.433 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:12.435 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:12.520 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] pass test: set log level [2024/05/06 13:30:12.933 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:13.118 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:13.126 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] check diff failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:12 GMT < Content-Length: 877 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/515731c7-6633-4261-85ae-58821f14dd2b {"id":"515731c7-6633-4261-85ae-58821f14dd2b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973410} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a0b4ad 515731c7-6633-4261-85ae-58821f14dd2b /tidb/cdc/default/default/upstream/7365754474880596156 {"id":7365754474880596156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/515731c7-6633-4261-85ae-58821f14dd2b {"id":"515731c7-6633-4261-85ae-58821f14dd2b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973410} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a0b4ad 515731c7-6633-4261-85ae-58821f14dd2b /tidb/cdc/default/default/upstream/7365754474880596156 {"id":7365754474880596156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/515731c7-6633-4261-85ae-58821f14dd2b {"id":"515731c7-6633-4261-85ae-58821f14dd2b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973410} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a0b4ad 515731c7-6633-4261-85ae-58821f14dd2b /tidb/cdc/default/default/upstream/7365754474880596156 {"id":7365754474880596156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6259.out cli changefeed create --start-ts=449569978795950081 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-2 [2024/05/06 13:30:13.434 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7365754474880596156,"namespace":"default","id":"changefeed-error-2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T13:30:13.387682528+08:00","start_ts":449569978795950081,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569978795950081,"checkpoint_ts":449569978795950081,"checkpoint_time":"2024-05-06 13:29:28.820"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 3-th time, retry later [2024/05/06 13:30:13.928 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:30:14.021 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:14.021 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:14.028 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:30:14.030 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:30:14.042 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:14.042 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:30:14.046 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:14.121 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:14.123 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:14.125 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:14.127 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 13:30:14.327 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:14.434 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:14.435 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:14.623 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s pass test: remove changefeed [2024/05/06 13:30:14.844 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:14.846 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:14.931 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:30:14.936 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:14.944 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:14.946 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:14.950 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:15.018 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:15.019 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:15.024 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] table test.finish_mark not exists for 9-th check, retry later check diff failed 4-th time, retry later + info='{ "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449569978795950081, "checkpoint_time": "2024-05-06 13:29:28.820", "error": { "time": "2024-05-06T13:30:13.468230661+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449569978795950081 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449569978795950081, "checkpoint_time": "2024-05-06 13:29:28.820", "error": { "time": "2024-05-06T13:30:13.468230661+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449569978795950081 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449569978795950081, "checkpoint_time": "2024-05-06 13:29:28.820", "error": { "time": "2024-05-06T13:30:13.468230661+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449569978795950081 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365754474880596156, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449569978795950081, '"checkpoint_time":' '"2024-05-06' '13:29:28.820",' '"error":' '{' '"time":' '"2024-05-06T13:30:13.468230661+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449569978795950081 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365754474880596156, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449569978795950081, '"checkpoint_time":' '"2024-05-06' '13:29:28.820",' '"error":' '{' '"time":' '"2024-05-06T13:30:13.468230661+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449569978795950081 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449569978795950081 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449569978795950081 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6348.out cli changefeed remove -c changefeed-error-2 [2024/05/06 13:30:15.126 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:30:15.229 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:15.230 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:30:15.430 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:15.536 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] check diff failed 4-th time, retry later [2024/05/06 13:30:15.825 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] pass test: resign owner Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc) 3730078720 bytes in 15.67 secs (238078465 bytes/sec) [Pipeline] { [Pipeline] sh [2024/05/06 13:30:15.934 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:16.029 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:16.035 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:16.039 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:30:16.043 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:16.044 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:16.124 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:30:16.127 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:30:16.132 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:30:16.133 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 449569978795950081 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:16.219 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:30:16.222 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:16.325 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:16.331 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [Pipeline] sh [2024/05/06 13:30:16.523 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G20 Run cases: tidb_mysql_test ddl_reentrant multi_cdc_cluster PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=8e2d2574-6afe-45f6-ab0b-fcd3609419e4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-855t7-fbskf-4lwp5 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G20 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-855t7 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1779-855t7 pingcap-tiflow-pull-cdc-integration-test-1779-855t7-fbskf-4lwp5 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-855t7-fbskf-4lwp5 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/tidb_mysql_test/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh [2024/05/06 13:30:16.730 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] table test.finish_mark not exists for 10-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G13 Run cases: tiflash region_merge common_1 PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=db7872bd-6bd5-480b-92e2-96b09e461557 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-st1k9-fprpv-trhhs HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-st1k9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1779-st1k9 pingcap-tiflow-pull-cdc-integration-test-1779-st1k9-fprpv-trhhs GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-st1k9-fprpv-trhhs GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh check diff failed 5-th time, retry later [2024/05/06 13:30:17.030 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:17.135 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G18 Run cases: availability http_proxies sequence PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=5f88b98a-bdfc-4177-ad14-e9fe976a5ba6 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-wn6pn-dr756-hx56v HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G18 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-wn6pn GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1779-wn6pn-dr756-hx56v pingcap_tiflow_pull_cdc_integration_test_1779-wn6pn GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-wn6pn-dr756-hx56v GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/availability/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh pass test: get tso [2024/05/06 13:30:17.226 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:17.226 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:30:17.241 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:17.322 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:17.327 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:17.334 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:30:17.334 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:30:17.344 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:30:17.432 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:17.432 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] + set +x + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G14 Run cases: changefeed_finish force_replicate_table PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=a9a5f08c-9657-4f4e-8ad0-abd62c712f6b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-dnsg9-wwwnp-xssfm HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-dnsg9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1779-dnsg9 pingcap-tiflow-pull-cdc-integration-test-1779-dnsg9-wwwnp-xssfm GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-dnsg9-wwwnp-xssfm GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh [2024/05/06 13:30:17.543 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:17.623 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] check diff failed 5-th time, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G15 Run cases: new_ci_collation batch_add_table multi_rocks PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=51778556-d8ca-4ed2-a42b-f46db82afdbb BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-63kf5-dnt52-vw515 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-63kf5 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1779-63kf5-dnt52-vw515 pingcap_tiflow_pull_cdc_integration_test_1779-63kf5 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-63kf5-dnt52-vw515 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh [2024/05/06 13:30:17.727 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:17.818 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G19 Run cases: changefeed_fast_fail batch_update_to_no_batch changefeed_resume_with_checkpoint_ts PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=c4051e35-188b-4b25-9b95-a2d3018dbf54 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-r8vbf-lj78v-9fw4x HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G19 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-r8vbf GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1779-r8vbf-lj78v-9fw4x pingcap_tiflow_pull_cdc_integration_test_1779-r8vbf GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-r8vbf-lj78v-9fw4x GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_fast_fail/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 13:30:18.023 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:18.122 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:18.128 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [Pipeline] sh pass test: verify table [2024/05/06 13:30:18.226 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:18.326 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:18.330 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:18.333 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:18.430 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G21 Run cases: bank kill_owner_with_ddl owner_remove_table_error PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=4be25d0e-e855-4508-b1e0-90744ef88098 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-0cr08-7wrc3-1z3gv HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G21 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-0cr08 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1779-0cr08 pingcap-tiflow-pull-cdc-integration-test-1779-0cr08-7wrc3-1z3gv GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-0cr08-7wrc3-1z3gv GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh wait process cdc.test exit for 2-th time... [2024/05/06 13:30:18.521 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:18.527 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:18.538 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:18.622 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G17 Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=1081dea8-fd05-4351-b0a5-a94d59387285 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-5gqnj-hmhkq-wjjv4 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-5gqnj GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1779-5gqnj pingcap-tiflow-pull-cdc-integration-test-1779-5gqnj-hmhkq-wjjv4 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-5gqnj-hmhkq-wjjv4 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table test.finish_mark not exists for 11-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:30:18 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.63956397.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 13:30:18.744 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:18.925 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] check diff successfully [2024/05/06 13:30:19.020 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:19.022 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:19.127 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:19.147 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:19.220 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:19.227 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G16 Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=8d234dcf-5a26-4bd1-99f4-ab34d5cf387d JENKINS_NODE_COOKIE=d3319067-3555-493c-8b02-c530dc325216 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1779-l2sg4-3h871-3slfr HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1779 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1787347784425279491","prowjobid":"8d234dcf-5a26-4bd1-99f4-ab34d5cf387d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11021,"author":"asddongmen","sha":"d72d260c017170db94deda602246fcddfea2a4a1","title":"sink (ticdc): fix sourceID error","link":"https://github.com/pingcap/tiflow/pull/11021","commit_link":"https://github.com/pingcap/tiflow/pull/11021/commits/d72d260c017170db94deda602246fcddfea2a4a1","author_link":"https://github.com/asddongmen"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1779 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787347784425279491 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1779/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1779-l2sg4 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1779-l2sg4-3h871-3slfr pingcap_tiflow_pull_cdc_integration_test_1779-l2sg4 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1779-l2sg4-3h871-3slfr GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1779 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 13:30:19.244 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:19.245 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:19.323 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:19.332 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:19.420 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:19.425 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:19.437 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:19.520 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:19.723 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] pass test: create changefeed v2 check diff successfully start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... [2024/05/06 13:30:20.123 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:20.129 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:20.132 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:20.219 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:20.236 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] wait process cdc.test exit for 1-th time... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/availability Starting Upstream PD... [2024/05/06 13:30:20.322 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:20.326 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:20.335 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:20.418 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:20.425 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:20.430 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:20.441 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 13:30:20.520 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:20.529 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:20.533 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 12-th check, retry later Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/06 13:30:20.631 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] The 1 times to try to start tidb cluster... check diff failed 1-th time, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:30:21 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> [2024/05/06 13:30:21.138 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:21.140 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:21.319 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:21.334 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] pass test: remove changefeed v2 [2024/05/06 13:30:21.420 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:21.420 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:21.446 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:21.532 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:21.535 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:21.545 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] Verifying downstream PD is started... [2024/05/06 13:30:21.835 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 13:30:21.838 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] start tidb cluster in /tmp/tidb_cdc_test/clustered_index Starting Upstream PD... start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/06 13:30:21.930 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 13:30:22.018 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 13:30:22.042 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 13:30:22.124 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:21 GMT < Content-Length: 877 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c402d799-be85-44e3-9603-0e1d043954af {"id":"c402d799-be85-44e3-9603-0e1d043954af","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973419} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a0b50e c402d799-be85-44e3-9603-0e1d043954af /tidb/cdc/default/default/upstream/7365754474880596156 {"id":7365754474880596156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c402d799-be85-44e3-9603-0e1d043954af {"id":"c402d799-be85-44e3-9603-0e1d043954af","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973419} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a0b50e c402d799-be85-44e3-9603-0e1d043954af /tidb/cdc/default/default/upstream/7365754474880596156 {"id":7365754474880596156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c402d799-be85-44e3-9603-0e1d043954af {"id":"c402d799-be85-44e3-9603-0e1d043954af","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973419} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f4c60a0b50e c402d799-be85-44e3-9603-0e1d043954af /tidb/cdc/default/default/upstream/7365754474880596156 {"id":7365754474880596156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6449.out cli changefeed create --start-ts=0 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-initialize-error Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_fast_fail Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/06 13:30:22.237 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 13:30:22.337 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 13:30:22.419 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7365754474880596156,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T13:30:22.412311197+08:00","start_ts":449569992820654083,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569992820654083,"checkpoint_ts":449569992820654083,"checkpoint_time":"2024-05-06 13:30:22.320"} PASS ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 13-th check, retry later [2024/05/06 13:30:22.431 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] coverage: 2.4% of statements in github.com/pingcap/tiflow/... status code 200 pass test: show metadata status code 200 status code 200 pass test: delete service_gc_safepoint status code 500 pass test: resolve lock [2024/05/06 13:30:22.749 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 13:30:22.786 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 13:30:22.831 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 13:30:22.837 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 13:30:23.033 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa08c8657_4808_4abd_b7db_26a42d88ce67"] [2024/05/06 13:30:23.125 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs26310bf2_91f3_4f10_998a_c075f3500e52"] Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Verifying downstream PD is started... check diff failed 2-th time, retry later [2024/05/06 13:30:23.221 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 13:30:23.425 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/06 13:30:23.628 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs19f46b24_565b_43a7_ad6c_b0330a548d5a"] [2024/05/06 13:30:23.767 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:23.768 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:23.820 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:23.829 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:23.924 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa52a7de5_9426_453e_a019_9dedf06429e5"] [2024/05/06 13:30:23.931 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd507b4ac_ab9e_4b06_ba82_51523ab810c6"] + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449569992820654083, "checkpoint_time": "2024-05-06 13:30:22.320", "error": { "time": "2024-05-06T13:30:22.590816586+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449569992820654083, "checkpoint_time": "2024-05-06 13:30:22.320", "error": { "time": "2024-05-06T13:30:22.590816586+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449569992820654083, "checkpoint_time": "2024-05-06 13:30:22.320", "error": { "time": "2024-05-06T13:30:22.590816586+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365754474880596156, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449569992820654083, '"checkpoint_time":' '"2024-05-06' '13:30:22.320",' '"error":' '{' '"time":' '"2024-05-06T13:30:22.590816586+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/06 13:30:23.967 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:23.970 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:24.047 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd20de147_dcc6_4bdc_835b_96acf5154dc7"] + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3186be140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57, pid:9968, start at 2024-05-06 13:30:21.482102462 +0800 CST m=+5.439297321 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:21.493 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:21.496 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:21.496 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3186be140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57, pid:9968, start at 2024-05-06 13:30:21.482102462 +0800 CST m=+5.439297321 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:21.493 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:21.496 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:21.496 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3186bec80009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57, pid:10044, start at 2024-05-06 13:30:21.499972024 +0800 CST m=+5.406852603 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:21.506 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:21.490 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:21.490 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 13:30:24.229 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:24.331 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/06 13:30:24.622 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs2ccafba2_5419_403b_bab1_5756c9775afc"] [2024/05/06 13:30:24.627 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs0badb8fd_e0d1_4f70_b54f_970b3337f9f8"] [2024/05/06 13:30:24.630 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:24.630 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 13:30:24.927 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:24.945 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] wait process cdc.test exit for 2-th time... table test.finish_mark not exists for 14-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:25.021 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:25.030 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:25.042 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:25.046 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:25.224 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:25.225 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] check diff failed 3-th time, retry later [2024/05/06 13:30:25.328 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:25.421 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 3-th time... [2024/05/06 13:30:25.528 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:25.541 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:25.546 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:25.624 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:25.628 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:25.632 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... [2024/05/06 13:30:25.827 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:25.934 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:30:25 CST 2024] <<<<<< run test case http_api_tls success! >>>>>> check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s [2024/05/06 13:30:26.132 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:26.133 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:26.221 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:26.225 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11306.out cli tso query --pd=http://127.0.0.1:2379 + info='{ "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449569992820654083, "checkpoint_time": "2024-05-06 13:30:22.320", "error": { "time": "2024-05-06T13:30:22.590816586+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449569992820654083, "checkpoint_time": "2024-05-06 13:30:22.320", "error": { "time": "2024-05-06T13:30:22.590816586+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449569992820654083, "checkpoint_time": "2024-05-06 13:30:22.320", "error": { "time": "2024-05-06T13:30:22.590816586+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365754474880596156, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449569992820654083, '"checkpoint_time":' '"2024-05-06' '13:30:22.320",' '"error":' '{' '"time":' '"2024-05-06T13:30:22.590816586+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 13:30:26.333 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:26.335 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:26.525 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] start tidb cluster in /tmp/tidb_cdc_test/bank Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/06 13:30:26.539 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:26.735 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... table test.finish_mark not exists for 15-th check, retry later [2024/05/06 13:30:26.824 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:26.826 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:26.827 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:26.828 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:26.838 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:27.020 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:27.036 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 13:30:27.137 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:27.233 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table testSync.usertable exists table testSync.simple1 exists table testSync.simple2 exists [2024/05/06 13:30:27.321 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:27.325 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:27.334 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:27.430 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:27.720 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:30:27.722 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 13:30:28.036 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:28.037 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:28.037 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:28.041 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449569993967534081 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449569993967534081 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449569993967534081 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Mon May 6 13:30:28 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1133611338.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 13:30:28.138 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:30:28.148 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:28.324 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:28.330 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:28.437 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:28.523 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:28.620 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:28.624 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:28.624 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:28.628 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:28.734 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:28.736 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] table test.finish_mark not exists for 16-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... [2024/05/06 13:30:29.034 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:29.120 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:29.127 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:29.133 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:29.324 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:29.328 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:29.435 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:29.535 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:29.644 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:29.742 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:29.746 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:29.842 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:29.929 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:30:29.933 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:30:30.019 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:30.022 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:30:30.136 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:30.145 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:30.146 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:30.234 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449569994655662086, "checkpoint_time": "2024-05-06 13:30:29.320", "error": null }' + echo '{ "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449569994655662086, "checkpoint_time": "2024-05-06 13:30:29.320", "error": null }' { "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449569994655662086, "checkpoint_time": "2024-05-06 13:30:29.320", "error": null } ++ echo '{' '"upstream_id":' 7365754474880596156, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449569994655662086, '"checkpoint_time":' '"2024-05-06' '13:30:29.320",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365754474880596156, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449569994655662086, '"checkpoint_time":' '"2024-05-06' '13:30:29.320",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6656.out cli changefeed pause -c changefeed-initialize-error [2024/05/06 13:30:30.428 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:30:30.429 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:30:30.531 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark exists check diff successfully [2024/05/06 13:30:30.739 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:30.832 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:30.943 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:31.046 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:31.220 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:31.231 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:31.318 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:31.320 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:31.328 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:31 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/916dde17-ae8a-4112-806e-ce0a93a5813d {"id":"916dde17-ae8a-4112-806e-ce0a93a5813d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973428} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61867dd3 916dde17-ae8a-4112-806e-ce0a93a5813d /tidb/cdc/default/default/upstream/7365754718963653504 {"id":7365754718963653504,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/916dde17-ae8a-4112-806e-ce0a93a5813d {"id":"916dde17-ae8a-4112-806e-ce0a93a5813d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973428} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61867dd3 916dde17-ae8a-4112-806e-ce0a93a5813d /tidb/cdc/default/default/upstream/7365754718963653504 {"id":7365754718963653504,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/916dde17-ae8a-4112-806e-ce0a93a5813d {"id":"916dde17-ae8a-4112-806e-ce0a93a5813d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973428} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61867dd3 916dde17-ae8a-4112-806e-ce0a93a5813d /tidb/cdc/default/default/upstream/7365754718963653504 {"id":7365754718963653504,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449569993967534081 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11389.out cli changefeed create --start-ts=449569993967534081 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml [2024/05/06 13:30:31.520 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:31.521 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:31.527 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:31.535 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:31.630 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:31.640 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365754718963653504,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T13:30:31.696863734+08:00","start_ts":449569993967534081,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569993967534081,"checkpoint_ts":449569993967534081,"checkpoint_time":"2024-05-06 13:30:26.695"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:31.824 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:31.934 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:32.128 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... [2024/05/06 13:30:32.327 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449569994917806086, "checkpoint_time": "2024-05-06 13:30:30.320", "error": { "time": "2024-05-06T13:30:22.590816586+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449569994917806086, "checkpoint_time": "2024-05-06 13:30:30.320", "error": { "time": "2024-05-06T13:30:22.590816586+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449569994917806086, "checkpoint_time": "2024-05-06 13:30:30.320", "error": { "time": "2024-05-06T13:30:22.590816586+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365754474880596156, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449569994917806086, '"checkpoint_time":' '"2024-05-06' '13:30:30.320",' '"error":' '{' '"time":' '"2024-05-06T13:30:22.590816586+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3187583c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-wn6pn-dr756-hx56v, pid:1250, start at 2024-05-06 13:30:31.332850198 +0800 CST m=+5.130973758 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:31.339 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:31.311 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:31.311 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3187583c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-wn6pn-dr756-hx56v, pid:1250, start at 2024-05-06 13:30:31.332850198 +0800 CST m=+5.130973758 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:31.339 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:31.311 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:31.311 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318759d00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-wn6pn-dr756-hx56v, pid:1343, start at 2024-05-06 13:30:31.417229306 +0800 CST m=+5.156172157 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:31.426 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:31.412 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:31.412 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/availability/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/availability/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/availability/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/availability/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/availability/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 13:30:32.519 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:30:32.622 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:32.631 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:32.721 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365754474880596156, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449569994917806086, '"checkpoint_time":' '"2024-05-06' '13:30:30.320",' '"error":' '{' '"time":' '"2024-05-06T13:30:22.590816586+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6744.out cli changefeed resume -c changefeed-initialize-error cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:30:32 CST 2024] <<<<<< run test case many_pk_or_uk success! >>>>>> [2024/05/06 13:30:32.736 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:30:32.737 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:32.829 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:32.839 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:32.841 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:32.923 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:30:32.940 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:30:32.941 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:33.028 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:33.119 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:30:33.227 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31874fdc0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-st1k9-fprpv-trhhs, pid:1234, start at 2024-05-06 13:30:30.819904045 +0800 CST m=+5.819838520 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:30.831 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:30.825 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:30.825 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31874fdc0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-st1k9-fprpv-trhhs, pid:1234, start at 2024-05-06 13:30:30.819904045 +0800 CST m=+5.819838520 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:30.831 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:30.825 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:30.825 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3187515c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-st1k9-fprpv-trhhs, pid:1309, start at 2024-05-06 13:30:30.89586934 +0800 CST m=+5.838957545 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:30.902 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:30.871 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:30.871 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 1-th time, retry later 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 5155 0 --:--:-- --:--:-- --:--:-- 5261 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 13:30:26.695","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-06 13:30:33.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '13:30:26.695","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '13:30:33.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '13:30:26.695","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '13:30:33.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-06 13:30:26.695' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '13:30:26.695","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '13:30:33.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '13:30:26.695","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '13:30:33.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-06 13:30:33' + echo 'sink_checkpoint_ts is 2024-05-06' 13:30:26.695 sink_checkpoint_ts is 2024-05-06 13:30:26.695 ++ date -d '2024-05-06 13:30:26.695' +%s + checkpoint_timestamp=1714973426 ++ date -d '2024-05-06 13:30:33' +%s + current_timestamp=1714973433 + '[' 7 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' [2024/05/06 13:30:33.420 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:33.537 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:33.838 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:33.919 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:30:33.927 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:33.935 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:34.023 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:34.027 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:30:34.037 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:34.039 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:34.128 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:30:34.146 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:30:34.147 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:34.240 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:34.242 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318774840013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-r8vbf-lj78v-9fw4x, pid:1232, start at 2024-05-06 13:30:33.149113399 +0800 CST m=+5.335450380 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:33.156 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:33.121 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:33.121 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:34.345 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:34.435 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:27.096 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/06 13:30:27.096 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7365754086620651211] [2024/05/06 13:30:27.096 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 13:30:27.096 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/06 13:30:27.097 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/06 13:30:27.097 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 13:30:27.098 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/06 13:30:27.098 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7365754086620651211] [2024/05/06 13:30:27.098 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 13:30:27.098 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/06 13:30:27.098 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/06 13:30:27.098 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 13:30:27.099 +08:00] [INFO] [tikv_driver.go:197] ["using API V1."] [2024/05/06 13:30:27.099 +08:00] [INFO] [main.go:180] ["genLock started"] [2024/05/06 13:30:27.100 +08:00] [INFO] [store_cache.go:477] ["change store resolve state"] [store=1] [addr=127.0.0.1:20161] [from=unresolved] [to=resolved] [liveness-state=reachable] [2024/05/06 13:30:34.532 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:34.829 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:34.836 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:34.931 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:35.031 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.2651.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449569994917806086, "checkpoint_time": "2024-05-06 13:30:30.320", "error": null }' + echo '{ "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449569994917806086, "checkpoint_time": "2024-05-06 13:30:30.320", "error": null }' { "upstream_id": 7365754474880596156, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449569994917806086, "checkpoint_time": "2024-05-06 13:30:30.320", "error": null } ++ echo '{' '"upstream_id":' 7365754474880596156, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449569994917806086, '"checkpoint_time":' '"2024-05-06' '13:30:30.320",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365754474880596156, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449569994917806086, '"checkpoint_time":' '"2024-05-06' '13:30:30.320",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6829.out cli changefeed remove -c changefeed-initialize-error [2024/05/06 13:30:35.130 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:35.132 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:35.230 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:35.232 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] check diff failed 2-th time, retry later [2024/05/06 13:30:35.321 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:35.338 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:35.419 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:35.442 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:35.442 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:35.625 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:35.732 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] table test.t1 exists + sleep 5 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2635.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3187717c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-63kf5-dnt52-vw515, pid:1236, start at 2024-05-06 13:30:32.957405804 +0800 CST m=+5.341424241 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:32.963 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:32.927 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:32.927 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3187717c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-63kf5-dnt52-vw515, pid:1236, start at 2024-05-06 13:30:32.957405804 +0800 CST m=+5.341424241 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:32.963 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:32.927 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:32.927 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318774980002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-63kf5-dnt52-vw515, pid:1308, start at 2024-05-06 13:30:33.127589177 +0800 CST m=+5.454923570 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:33.137 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:33.126 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:33.126 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 13:30:35.826 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:35.942 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:35.946 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:36.034 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/api_v2/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 449569994917806086 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318774840013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-r8vbf-lj78v-9fw4x, pid:1232, start at 2024-05-06 13:30:33.149113399 +0800 CST m=+5.335450380 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:33.156 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:33.121 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:33.121 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3187760c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-r8vbf-lj78v-9fw4x, pid:1321, start at 2024-05-06 13:30:33.228359372 +0800 CST m=+5.359213986 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:33.237 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:33.219 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:33.219 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 13:30:36.133 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:36.219 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:36.226 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:36.319 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:36.321 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] + set +x + tso='449569996095094785 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449569996095094785 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [2024/05/06 13:30:36.323 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:36.418 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:36.423 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:36.523 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:36.532 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] run test case test_kill_owner [Mon May 6 13:30:36 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.27182720.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318790680015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-l2sg4-3h871-3slfr, pid:1297, start at 2024-05-06 13:30:34.936815704 +0800 CST m=+5.231220224 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:34.943 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:34.906 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:34.906 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318790680015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-l2sg4-3h871-3slfr, pid:1297, start at 2024-05-06 13:30:34.936815704 +0800 CST m=+5.231220224 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:34.943 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:34.906 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:34.906 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3187a1940010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-l2sg4-3h871-3slfr, pid:1382, start at 2024-05-06 13:30:36.020727796 +0800 CST m=+6.260828011 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:36.026 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:36.005 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:36.005 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_resign/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_resign/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 13:30:36.730 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31879e74000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-0cr08-7wrc3-1z3gv, pid:1238, start at 2024-05-06 13:30:35.815768805 +0800 CST m=+5.213656133 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:35.822 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:35.805 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:35.805 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:36.921 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:36.925 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] check diff failed 3-th time, retry later + set +x + tso='449569996321587201 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449569996321587201 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:30:37 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.26742676.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 13:30:37.121 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:37.229 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:37.234 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:37.320 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:37.323 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:37.328 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 13:30:37.332 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:37.336 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31878d800006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-855t7-fbskf-4lwp5, pid:1310, start at 2024-05-06 13:30:34.729906084 +0800 CST m=+5.710877527 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:34.742 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:34.720 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:34.720 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31878d800006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-855t7-fbskf-4lwp5, pid:1310, start at 2024-05-06 13:30:34.729906084 +0800 CST m=+5.710877527 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:34.742 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:34.720 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:34.720 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31878e540015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-855t7-fbskf-4lwp5, pid:1386, start at 2024-05-06 13:30:34.80936716 +0800 CST m=+5.722441205 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:34.815 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:34.823 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:34.823 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3187894c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-dnsg9-wwwnp-xssfm, pid:1294, start at 2024-05-06 13:30:34.466551911 +0800 CST m=+5.536005765 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:34.472 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:34.451 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:34.451 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3187894c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-dnsg9-wwwnp-xssfm, pid:1294, start at 2024-05-06 13:30:34.466551911 +0800 CST m=+5.536005765 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:34.472 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:34.451 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:34.451 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31878ad00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-dnsg9-wwwnp-xssfm, pid:1377, start at 2024-05-06 13:30:34.581191862 +0800 CST m=+5.572857986 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:34.587 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:34.548 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:34.548 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 13:30:37.528 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:37.535 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/06 13:30:37.103 +08:00] [INFO] [main.go:196] ["genLock done"] [2024/05/06 13:30:37.103 +08:00] [INFO] [pd_service_discovery.go:550] ["[pd] exit member loop due to context canceled"] [2024/05/06 13:30:37.103 +08:00] [INFO] [resource_manager_client.go:295] ["[resource manager] exit resource token dispatcher"] [2024/05/06 13:30:37.103 +08:00] [INFO] [tso_dispatcher.go:214] ["exit tso requests cancel loop"] [2024/05/06 13:30:37.103 +08:00] [INFO] [tso_dispatcher.go:455] ["[tso] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2024/05/06 13:30:37.103 +08:00] [INFO] [tso_client.go:140] ["closing tso client"] [2024/05/06 13:30:37.103 +08:00] [INFO] [tso_dispatcher.go:268] ["exit tso dispatcher loop"] [2024/05/06 13:30:37.103 +08:00] [INFO] [tso_dispatcher.go:380] ["[tso] exit tso dispatcher"] [dc-location=global] [2024/05/06 13:30:37.103 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/06 13:30:37.103 +08:00] [INFO] [tso_client.go:145] ["close tso client"] [2024/05/06 13:30:37.103 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/06 13:30:37.103 +08:00] [INFO] [tso_client.go:155] ["tso client is closed"] [2024/05/06 13:30:37.103 +08:00] [INFO] [pd_service_discovery.go:637] ["[pd] close pd service discovery client"] [2024/05/06 13:30:37.825 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] wait process cdc.test exit for 1-th time... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.2635.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 2-th time... [2024/05/06 13:30:38.129 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:38.345 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:38.364 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [Mon May 6 13:30:38 CST 2024] <<<<<< START cdc server in new_ci_collation case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation.26532655.out server --log-file /tmp/tidb_cdc_test/new_ci_collation/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 13:30:38 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.26292631.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 13:30:38.431 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:38.435 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:38.435 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:38.552 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31879e74000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-0cr08-7wrc3-1z3gv, pid:1238, start at 2024-05-06 13:30:35.815768805 +0800 CST m=+5.213656133 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:35.822 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:35.805 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:35.805 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3187a078000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-0cr08-7wrc3-1z3gv, pid:1318, start at 2024-05-06 13:30:35.946236729 +0800 CST m=+5.286309328 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:35.956 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:35.934 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:35.934 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/05/06 13:30:38.821 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 13:30:38.922 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs29e2196e_70cb_41db_87e4_8b110f2d0fc1"] wait process cdc.test exit for 3-th time... Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bank/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bank/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/bank/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/bank/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/bank/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 13:30:39.130 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 13:30:39.137 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3187b75c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5gqnj-hmhkq-wjjv4, pid:1211, start at 2024-05-06 13:30:37.405043409 +0800 CST m=+10.154172602 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:37.450 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:37.548 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:37.548 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3187b75c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5gqnj-hmhkq-wjjv4, pid:1211, start at 2024-05-06 13:30:37.405043409 +0800 CST m=+10.154172602 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:37.450 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:37.548 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:37.548 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3187bca00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5gqnj-hmhkq-wjjv4, pid:1279, start at 2024-05-06 13:30:37.739737385 +0800 CST m=+10.431101257 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:37.754 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:37.736 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:37.736 +0800 All versions after safe point can be accessed. (DO NOT EDIT) start tidb cluster in /tmp/tidb_cdc_test/api_v2 Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/06 13:30:39.260 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 13:30:39.426 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:39.428 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.cli.2690.out cli tso query --pd=http://127.0.0.1:2379 [Mon May 6 13:30:39 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.26432645.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:30:39 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... [2024/05/06 13:30:39.525 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 13:30:39.528 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs01ccbb8f_66b0_4668_b907_a637c3b8030f"] [2024/05/06 13:30:39.530 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs0a796d26_839f_44a3_be35_99569946fd71"] [2024/05/06 13:30:39.533 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] check diff failed 4-th time, retry later Logging trace to /tmp/tidb_cdc_test/clustered_index/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/clustered_index/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:39 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09 {"id":"5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973436} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3df5 5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09 /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09 {"id":"5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973436} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3df5 5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09 /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09 {"id":"5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973436} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3df5 5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09 /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.2779.out cli changefeed create --start-ts=449569996095094785 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ [2024/05/06 13:30:39.736 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs226a7506_92bf_4507_be4c_883f4560991a"] [2024/05/06 13:30:39.826 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs39b1634b_ffc3_4822_84ed_c2bf4244f22b"] [2024/05/06 13:30:39.930 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:39.930 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] + set +x + tso='449569997041434625 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449569997041434625 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:30:39 CST 2024] <<<<<< START cdc server in changefeed_fast_fail case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.26792681.out server --log-file /tmp/tidb_cdc_test/changefeed_fast_fail/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_fast_fail/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: 76bfc1da-dc68-45bf-ac43-3f92d952b920 Info: {"upstream_id":7365754766099386827,"namespace":"default","id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:30:40.10650401+08:00","start_ts":449569996095094785,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569996095094785,"checkpoint_ts":449569996095094785,"checkpoint_time":"2024-05-06 13:30:34.811"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/06 13:30:40.023 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsf78a99a4_dd7a_45f1_8654_cff0997aa263"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 13:30:40.420 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs7ed1cb42_b420_4566_837e_528cfa19dd6a"] [2024/05/06 13:30:40.423 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:40.425 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:40.428 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:40.429 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:40 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d4ba6ab5-2b27-432f-8f23-a485b684ec20 {"id":"d4ba6ab5-2b27-432f-8f23-a485b684ec20","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973437} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ab53d2 d4ba6ab5-2b27-432f-8f23-a485b684ec20 /tidb/cdc/default/default/upstream/7365754755662041814 {"id":7365754755662041814,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d4ba6ab5-2b27-432f-8f23-a485b684ec20 {"id":"d4ba6ab5-2b27-432f-8f23-a485b684ec20","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973437} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ab53d2 d4ba6ab5-2b27-432f-8f23-a485b684ec20 /tidb/cdc/default/default/upstream/7365754755662041814 {"id":7365754755662041814,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d4ba6ab5-2b27-432f-8f23-a485b684ec20 {"id":"d4ba6ab5-2b27-432f-8f23-a485b684ec20","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973437} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ab53d2 d4ba6ab5-2b27-432f-8f23-a485b684ec20 /tidb/cdc/default/default/upstream/7365754755662041814 {"id":7365754755662041814,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 7d2473d2-87eb-4ecb-a151-27eda8795f0e Info: {"upstream_id":7365754755662041814,"namespace":"default","id":"7d2473d2-87eb-4ecb-a151-27eda8795f0e","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:30:40.472208639+08:00","start_ts":449569996321587201,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569996321587201,"checkpoint_ts":449569996321587201,"checkpoint_time":"2024-05-06 13:30:35.675"} [2024/05/06 13:30:40.643 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:40.724 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:40.736 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:40.820 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:40.835 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:40.840 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:40.928 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2538 0 --:--:-- --:--:-- --:--:-- 2557 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 13:30:39.096","puller_resolved_ts":"2024-05-06 13:30:33.396","last_synced_ts":"2024-05-06 13:30:33.846","now_ts":"2024-05-06 13:30:40.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '13:30:39.096","puller_resolved_ts":"2024-05-06' '13:30:33.396","last_synced_ts":"2024-05-06' '13:30:33.846","now_ts":"2024-05-06' '13:30:40.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '13:30:39.096","puller_resolved_ts":"2024-05-06' '13:30:33.396","last_synced_ts":"2024-05-06' '13:30:33.846","now_ts":"2024-05-06' '13:30:40.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 [2024/05/06 13:30:41.018 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] + set +x + tso='449569997382221825 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449569997382221825 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:30:41 CST 2024] <<<<<< START cdc server in tidb_mysql_test case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.27262728.out server --log-file /tmp/tidb_cdc_test/tidb_mysql_test/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tidb_mysql_test/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 13:30:41 CST 2024] <<<<<< START cdc server in bank case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.26482650.out server --log-file /tmp/tidb_cdc_test/bank/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bank/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' check diff failed 5-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:41 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1c26024-7c7c-4811-b1c0-2428181932cf {"id":"c1c26024-7c7c-4811-b1c0-2428181932cf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973438} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61afa0d2 c1c26024-7c7c-4811-b1c0-2428181932cf /tidb/cdc/default/default/upstream/7365754758254926277 {"id":7365754758254926277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1c26024-7c7c-4811-b1c0-2428181932cf {"id":"c1c26024-7c7c-4811-b1c0-2428181932cf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973438} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61afa0d2 c1c26024-7c7c-4811-b1c0-2428181932cf /tidb/cdc/default/default/upstream/7365754758254926277 {"id":7365754758254926277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1c26024-7c7c-4811-b1c0-2428181932cf {"id":"c1c26024-7c7c-4811-b1c0-2428181932cf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973438} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61afa0d2 c1c26024-7c7c-4811-b1c0-2428181932cf /tidb/cdc/default/default/upstream/7365754758254926277 {"id":7365754758254926277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 92732170-1457-4573-98fa-51f3bc072bf8 Info: {"upstream_id":7365754758254926277,"namespace":"default","id":"92732170-1457-4573-98fa-51f3bc072bf8","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-05-06T13:30:41.326105232+08:00","start_ts":449569996938149890,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569996938149890,"checkpoint_ts":449569996938149890,"checkpoint_time":"2024-05-06 13:30:38.027"} [2024/05/06 13:30:41.535 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:41.535 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:41.620 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:41.621 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:41.623 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:41.626 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:41 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e391a80-79b4-476f-bb83-76e6613758cf {"id":"8e391a80-79b4-476f-bb83-76e6613758cf","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973438} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b9e5c7 8e391a80-79b4-476f-bb83-76e6613758cf /tidb/cdc/default/default/upstream/7365754778751316948 {"id":7365754778751316948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e391a80-79b4-476f-bb83-76e6613758cf {"id":"8e391a80-79b4-476f-bb83-76e6613758cf","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973438} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b9e5c7 8e391a80-79b4-476f-bb83-76e6613758cf /tidb/cdc/default/default/upstream/7365754778751316948 {"id":7365754778751316948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e391a80-79b4-476f-bb83-76e6613758cf {"id":"8e391a80-79b4-476f-bb83-76e6613758cf","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973438} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b9e5c7 8e391a80-79b4-476f-bb83-76e6613758cf /tidb/cdc/default/default/upstream/7365754778751316948 {"id":7365754778751316948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.cli.2732.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 [2024/05/06 13:30:41.942 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:42.021 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:42.039 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:42.049 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:42.126 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:42.136 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:42.150 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:42.153 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [Mon May 6 13:30:41 CST 2024] <<<<<< START cdc server in clustered_index case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.clustered_index.26182620.out server --log-file /tmp/tidb_cdc_test/clustered_index/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/clustered_index/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: ebb10e78-57a7-438e-9346-3c9a7ba93b5c Info: {"upstream_id":7365754778751316948,"namespace":"default","id":"ebb10e78-57a7-438e-9346-3c9a7ba93b5c","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:30:42.123188664+08:00","start_ts":449569997981483009,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569997981483009,"checkpoint_ts":449569997981483009,"checkpoint_time":"2024-05-06 13:30:42.007"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later [2024/05/06 13:30:42.522 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa0f446c8_625f_403e_a8ad_4ada31c62a90"] [2024/05/06 13:30:42.523 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:42.524 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:42.525 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:42 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f1e8f59-7518-4d45-a3ab-2b0ab4c994b3 {"id":"4f1e8f59-7518-4d45-a3ab-2b0ab4c994b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973439} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b8b4cf 4f1e8f59-7518-4d45-a3ab-2b0ab4c994b3 /tidb/cdc/default/default/upstream/7365754773601530271 {"id":7365754773601530271,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f1e8f59-7518-4d45-a3ab-2b0ab4c994b3 {"id":"4f1e8f59-7518-4d45-a3ab-2b0ab4c994b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973439} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b8b4cf 4f1e8f59-7518-4d45-a3ab-2b0ab4c994b3 /tidb/cdc/default/default/upstream/7365754773601530271 {"id":7365754773601530271,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f1e8f59-7518-4d45-a3ab-2b0ab4c994b3 {"id":"4f1e8f59-7518-4d45-a3ab-2b0ab4c994b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973439} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b8b4cf 4f1e8f59-7518-4d45-a3ab-2b0ab4c994b3 /tidb/cdc/default/default/upstream/7365754773601530271 {"id":7365754773601530271,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/06 13:30:42.633 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:42.637 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:42.719 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] table test.t2 not exists for 1-th check, retry later [2024/05/06 13:30:42.937 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:43.043 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:43 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/296fbcc7-65dd-4fa0-a160-288139e6f011 {"id":"296fbcc7-65dd-4fa0-a160-288139e6f011","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973440} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b094e2 296fbcc7-65dd-4fa0-a160-288139e6f011 /tidb/cdc/default/default/upstream/7365754770617725758 {"id":7365754770617725758,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/296fbcc7-65dd-4fa0-a160-288139e6f011 {"id":"296fbcc7-65dd-4fa0-a160-288139e6f011","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973440} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b094e2 296fbcc7-65dd-4fa0-a160-288139e6f011 /tidb/cdc/default/default/upstream/7365754770617725758 {"id":7365754770617725758,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/296fbcc7-65dd-4fa0-a160-288139e6f011 {"id":"296fbcc7-65dd-4fa0-a160-288139e6f011","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973440} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b094e2 296fbcc7-65dd-4fa0-a160-288139e6f011 /tidb/cdc/default/default/upstream/7365754770617725758 {"id":7365754770617725758,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.2739.out cli changefeed create --start-ts=449569997041434625 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-fast-fail [2024/05/06 13:30:43.124 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:43.140 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:43.223 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:43.236 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:30:43.237 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:43.334 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] "is-owner": true, run task successfully check diff successfully [2024/05/06 13:30:43.530 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:43.531 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:43.536 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:43.539 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:43.540 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] + set +x Create changefeed successfully! ID: changefeed-fast-fail Info: {"upstream_id":7365754770617725758,"namespace":"default","id":"changefeed-fast-fail","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T13:30:43.526926612+08:00","start_ts":449569997041434625,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569997041434625,"checkpoint_ts":449569997041434625,"checkpoint_time":"2024-05-06 13:30:38.421"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/06 13:30:43.621 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:43.626 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:43.630 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:43.826 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:43.841 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later [2024/05/06 13:30:43.934 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:44.027 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:44.135 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:44.218 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:44.233 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:44.334 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] table cdc_tiflash_test.multi_data_type exists check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:44 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e701ad9-f9f8-488c-8f57-cbdd61b3a349 {"id":"4e701ad9-f9f8-488c-8f57-cbdd61b3a349","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973441} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b735d3 4e701ad9-f9f8-488c-8f57-cbdd61b3a349 /tidb/cdc/default/default/upstream/7365754778086429681 {"id":7365754778086429681,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e701ad9-f9f8-488c-8f57-cbdd61b3a349 {"id":"4e701ad9-f9f8-488c-8f57-cbdd61b3a349","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973441} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b735d3 4e701ad9-f9f8-488c-8f57-cbdd61b3a349 /tidb/cdc/default/default/upstream/7365754778086429681 {"id":7365754778086429681,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e701ad9-f9f8-488c-8f57-cbdd61b3a349 {"id":"4e701ad9-f9f8-488c-8f57-cbdd61b3a349","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973441} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b735d3 4e701ad9-f9f8-488c-8f57-cbdd61b3a349 /tidb/cdc/default/default/upstream/7365754778086429681 {"id":7365754778086429681,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 142 0 0 100 142 0 1472 --:--:-- --:--:-- --:--:-- 1479 mysql_test start go: downloading github.com/maxshuang/mysql-tester v0.0.2 [2024/05/06 13:30:44.637 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:44.640 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:44 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9253b7c0-ac03-4084-8209-99ad1059e8ba {"id":"9253b7c0-ac03-4084-8209-99ad1059e8ba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973441} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61bf4cdb 9253b7c0-ac03-4084-8209-99ad1059e8ba /tidb/cdc/default/default/upstream/7365754782998276313 {"id":7365754782998276313,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9253b7c0-ac03-4084-8209-99ad1059e8ba {"id":"9253b7c0-ac03-4084-8209-99ad1059e8ba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973441} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61bf4cdb 9253b7c0-ac03-4084-8209-99ad1059e8ba /tidb/cdc/default/default/upstream/7365754782998276313 {"id":7365754782998276313,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9253b7c0-ac03-4084-8209-99ad1059e8ba {"id":"9253b7c0-ac03-4084-8209-99ad1059e8ba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973441} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61bf4cdb 9253b7c0-ac03-4084-8209-99ad1059e8ba /tidb/cdc/default/default/upstream/7365754782998276313 {"id":7365754782998276313,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.cli.2709.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ Create changefeed successfully! ID: d5d76eb2-4336-4eaa-a347-174d590798cc Info: {"upstream_id":7365754782998276313,"namespace":"default","id":"d5d76eb2-4336-4eaa-a347-174d590798cc","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-06T13:30:44.616879584+08:00","start_ts":449569998636318725,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569998636318725,"checkpoint_ts":449569998636318725,"checkpoint_time":"2024-05-06 13:30:44.505"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... owner pid: 2723 owner id 5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09 [Mon May 6 13:30:44 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.28812883.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 13:30:44.723 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:44.724 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:44.733 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:44.744 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:30:44.746 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:44.824 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:44 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a6ec203-5059-4293-ad99-69182b5262de {"id":"5a6ec203-5059-4293-ad99-69182b5262de","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973442} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b23ed9 5a6ec203-5059-4293-ad99-69182b5262de /tidb/cdc/default/default/upstream/7365754772585954946 {"id":7365754772585954946,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a6ec203-5059-4293-ad99-69182b5262de {"id":"5a6ec203-5059-4293-ad99-69182b5262de","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973442} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b23ed9 5a6ec203-5059-4293-ad99-69182b5262de /tidb/cdc/default/default/upstream/7365754772585954946 {"id":7365754772585954946,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a6ec203-5059-4293-ad99-69182b5262de {"id":"5a6ec203-5059-4293-ad99-69182b5262de","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973442} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b23ed9 5a6ec203-5059-4293-ad99-69182b5262de /tidb/cdc/default/default/upstream/7365754772585954946 {"id":7365754772585954946,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table test.t2 not exists for 2-th check, retry later 1 run task successfully table owner_resign.t1 not exists for 1-th check, retry later [2024/05/06 13:30:45.020 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:45.025 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:30:45.034 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:45.118 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:30:45.140 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:30:45.143 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] Create changefeed successfully! ID: 0f37c68c-150b-4466-89f3-ae84eab56420 Info: {"upstream_id":7365754772585954946,"namespace":"default","id":"0f37c68c-150b-4466-89f3-ae84eab56420","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:30:45.105303488+08:00","start_ts":449569997913587713,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449569997913587713,"checkpoint_ts":449569997913587713,"checkpoint_time":"2024-05-06 13:30:41.748"} + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-fast-fail failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-fast-fail + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-fast-fail -s + info='{ "upstream_id": 7365754770617725758, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 449569997041434625, "checkpoint_time": "2024-05-06 13:30:38.421", "error": { "time": "2024-05-06T13:30:43.540617799+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7365754770617725758, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 449569997041434625, "checkpoint_time": "2024-05-06 13:30:38.421", "error": { "time": "2024-05-06T13:30:43.540617799+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7365754770617725758, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 449569997041434625, "checkpoint_time": "2024-05-06 13:30:38.421", "error": { "time": "2024-05-06T13:30:43.540617799+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ echo '{' '"upstream_id":' 7365754770617725758, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 449569997041434625, '"checkpoint_time":' '"2024-05-06' '13:30:38.421",' '"error":' '{' '"time":' '"2024-05-06T13:30:43.540617799+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' ++ jq -r .state wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_ddl/sync_diff/output" source-instances = ["tidb"] target-instance = "mysql" target-check-tables = ["consistent_replicate_ddl.usertable*","consistent_replicate_ddl.check*"] [data-sources] [data-sources.tidb] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "449569963609423877" [data-sources.mysql] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/05/06 13:30:33.610 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 13:30:33.611 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2024/05/06 13:30:33.611 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=:6060] [2024/05/06 13:30:33.611 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 13:30:33.611 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449569961696034833] [resolvedTs=449569963609423877] [2024/05/06 13:30:33.612 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 13:30:33.612 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 13:30:33.612 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2024/05/06 13:30:33.614 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=3.058409ms] [2024/05/06 13:30:33.615 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=3.124264ms] [2024/05/06 13:30:33.619 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 13:30:33.623 +08:00] [INFO] [mysql.go:197] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 13:30:33.623 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:30:33.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:30:33.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:30:33.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:30:33.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:30:33.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:30:33.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:30:33.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:30:33.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:30:33.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:30:33.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:30:33.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:30:33.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:30:33.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:30:33.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:30:33.624 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:30:33.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 13:30:33.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 13:30:33.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 13:30:33.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 13:30:33.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 13:30:33.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 13:30:33.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 13:30:33.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 13:30:33.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 13:30:33.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 13:30:33.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 13:30:33.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 13:30:33.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 13:30:33.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 13:30:33.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 13:30:33.624 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 13:30:33.624 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 13:30:33.624 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 13:30:33.624 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2024/05/06 13:30:33.628 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 13:30:33.631 +08:00] [INFO] [mysql_ddl_sink.go:107] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/06 13:30:33.631 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449569961696034827,\"CommitTs\":449569961696034833,\"Query\":\"DROP TABLE `consistent_replicate_ddl`.`usertable1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable1\",\"TableID\":108,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":4,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 13:30:33.631 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="DROP TABLE `consistent_replicate_ddl`.`usertable1`"] [commitTs=449569961696034833] [namespace=] [changefeed=] [2024/05/06 13:30:33.839 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="DROP TABLE `consistent_replicate_ddl`.`usertable1`"] [duration=207.320618ms] [namespace=] [changefeed=] [2024/05/06 13:30:33.839 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449569961853321223,\"CommitTs\":449569961853321227,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable1\",\"TableID\":114,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 13:30:33.839 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1"] [commitTs=449569961853321227] [namespace=] [changefeed=] [2024/05/06 13:30:33.980 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1"] [duration=141.645714ms] [namespace=] [changefeed=] [2024/05/06 13:30:33.981 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449569961957916681,\"CommitTs\":449569961957916685,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable1_1\",\"TableID\":114,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 13:30:34.182 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1"] [commitTs=449569961957916685] [namespace=] [changefeed=] [2024/05/06 13:30:34.323 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1"] [duration=141.337147ms] [namespace=] [changefeed=] [2024/05/06 13:30:34.323 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449569962075881506,\"CommitTs\":449569962088988673,\"Query\":\"CREATE DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl1\",\"Table\":\"\",\"TableID\":0,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":1,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 13:30:34.524 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="CREATE DATABASE consistent_replicate_ddl1"] [commitTs=449569962088988673] [namespace=] [changefeed=] [2024/05/06 13:30:34.603 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="CREATE DATABASE consistent_replicate_ddl1"] [duration=78.809553ms] [namespace=] [changefeed=] [2024/05/06 13:30:34.603 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449569962220060698,\"CommitTs\":449569962233167873,\"Query\":\"ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"\",\"TableID\":0,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":26,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 13:30:34.603 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [commitTs=449569962233167873] [namespace=] [changefeed=] [2024/05/06 13:30:34.678 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [duration=75.546082ms] [namespace=] [changefeed=] [2024/05/06 13:30:34.678 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449569962364239892,\"CommitTs\":449569962364239896,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 13:30:34.679 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [commitTs=449569962364239896] [namespace=] [changefeed=] [2024/05/06 13:30:34.757 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [duration=78.142302ms] [namespace=] [changefeed=] [2024/05/06 13:30:34.757 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449569962809884679,\"CommitTs\":449569962809884682,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":6,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 13:30:34.957 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0"] [commitTs=449569962809884682] [namespace=] [changefeed=] [2024/05/06 13:30:35.221 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0"] [duration=264.056735ms] [namespace=] [changefeed=] [2024/05/06 13:30:35.222 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449569962941218826,\"CommitTs\":449569962954063873,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":5,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 13:30:35.222 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)"] [commitTs=449569962954063873] [namespace=] [changefeed=] [2024/05/06 13:30:35.478 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)"] [duration=256.464146ms] [namespace=] [changefeed=] [2024/05/06 13:30:35.478 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449569963072290820,\"CommitTs\":449569963072290824,\"Query\":\"DROP DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl1\",\"Table\":\"\",\"TableID\":0,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":2,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 13:30:35.478 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="DROP DATABASE consistent_replicate_ddl1"] [commitTs=449569963072290824] [namespace=] [changefeed=] [2024/05/06 13:30:35.674 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="DROP DATABASE consistent_replicate_ddl1"] [duration=195.521928ms] [namespace=] [changefeed=] [2024/05/06 13:30:35.674 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449569963177148424,\"CommitTs\":449569963177148427,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable3\",\"TableID\":112,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 13:30:35.674 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"] [commitTs=449569963177148427] [namespace=] [changefeed=] [2024/05/06 13:30:35.750 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"] [duration=76.553779ms] [namespace=] [changefeed=] [2024/05/06 13:30:35.751 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449569963334172678,\"CommitTs\":449569963334172682,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2_1\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 13:30:35.751 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`"] [commitTs=449569963334172682] [namespace=] [changefeed=] [2024/05/06 13:30:35.885 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`"] [duration=134.729541ms] [namespace=] [changefeed=] [2024/05/06 13:30:35.885 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449569963334172678,\"CommitTs\":449569963334172682,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable3_1\",\"TableID\":112,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 13:30:35.885 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`"] [commitTs=449569963334172682] [namespace=] [changefeed=] [2024/05/06 13:30:36.020 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`"] [duration=134.602325ms] [namespace=] [changefeed=] [2024/05/06 13:30:36.020 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449569963452137491,\"CommitTs\":449569963452137495,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable3_1\",\"TableID\":112,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":12,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 13:30:36.020 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)"] [commitTs=449569963452137495] [namespace=] [changefeed=] [2024/05/06 13:30:36.095 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)"] [duration=75.219343ms] [namespace=] [changefeed=] [2024/05/06 13:30:36.096 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449569963596316703,\"CommitTs\":449569963609423877,\"Query\":\"CREATE table consistent_replicate_ddl.check1(id int primary key)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"check1\",\"TableID\":129,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 13:30:36.297 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="CREATE table consistent_replicate_ddl.check1(id int primary key)"] [commitTs=449569963609423877] [namespace=] [changefeed=] [2024/05/06 13:30:36.376 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="CREATE table consistent_replicate_ddl.check1(id int primary key)"] [duration=79.353902ms] [namespace=] [changefeed=] [2024/05/06 13:30:36.376 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449569963609423877] [2024/05/06 13:30:36.376 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449569963609423877] [2024/05/06 13:30:36.477 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}] [checkpointTs=449569963609423877] [2024/05/06 13:30:36.477 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}] [checkpointTs=449569963609423877] [2024/05/06 13:30:36.577 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}] [checkpointTs=449569963609423877] [2024/05/06 13:30:36.577 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}] [checkpointTs=449569963609423877] [2024/05/06 13:30:36.677 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=101] [appliedDDLCount=14] [currentCheckpoint=449569963609423877] [2024/05/06 13:30:36.677 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 13:30:36.677 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 13:30:36.677 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 13:30:36.677 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 13:30:36.677 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/06 13:30:36.677 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/06 13:30:36.677 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 13:30:36.677 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 13:30:36.677 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/06 13:30:36.677 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 13:30:36.677 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/06 13:30:36.677 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 13:30:36.677 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 13:30:36.677 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/06 13:30:36.677 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 13:30:36.677 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] Apply redo log successfully table consistent_replicate_ddl.check1 exists check diff successfully [Mon May 6 13:30:36 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> count(*) 50 [2024/05/06 13:30:45.224 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:45.233 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7365754770617725758, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 449569997041434625, '"checkpoint_time":' '"2024-05-06' '13:30:38.421",' '"error":' '{' '"time":' '"2024-05-06T13:30:43.540617799+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully [2024/05/06 13:30:45.525 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:45.530 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:45.531 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:45.537 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:45.635 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:30:45.636 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:45.724 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:45.730 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] check diff failed 1-th time, retry later + set +x + GO111MODULE=on + go run bank.go case.go -u 'root@tcp(127.0.0.1:4000)/bank' -d 'root@tcp(127.0.0.1:3306)/bank' --test-round=20000 -a 127.0.0.1:20080 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading go.uber.org/zap v1.27.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f [2024/05/06 13:30:46.035 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:46.124 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:46.136 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:46.147 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] go: downloading google.golang.org/grpc v1.62.1 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.uber.org/multierr v1.11.0 wait process cdc.test exit for 1-th time... [2024/05/06 13:30:46.240 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:30:46.244 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:46.329 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:46.339 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda [2024/05/06 13:30:46.531 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:46.535 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:46.535 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:46.621 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:46.624 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:46.628 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:46.632 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:30:46.646 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] table test.t2 not exists for 3-th check, retry later table owner_resign.t1 exists wait process cdc.test exit for 2-th time... check diff successfully wait process cdc.test exit for 1-th time... [2024/05/06 13:30:46.930 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:46.932 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] check diff failed 1-th time, retry later [2024/05/06 13:30:47.020 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:47.023 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:30:47.124 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:47.131 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:47.230 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:30:47.230 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:30:47 CST 2024] <<<<<< run test case changefeed_fast_fail success! >>>>>> wait process cdc.test exit for 2-th time... [2024/05/06 13:30:47.525 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:30:47.626 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:47.631 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:47.638 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:30:47.646 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:30:47.721 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 check diff failed 2-th time, retry later [2024/05/06 13:30:47.727 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:30:47.732 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:47.939 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b0a870 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449569996527894567 MinTableBarrierTs:449569999358787592 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53ad6578-f45e-425c-908e-6058bb0cbd6b {"id":"53ad6578-f45e-425c-908e-6058bb0cbd6b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973444} /tidb/cdc/default/__cdc_meta__/capture/5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09 {"id":"5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973436} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3df5 5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3e5b 53ad6578-f45e-425c-908e-6058bb0cbd6b /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449569996527894567,"min-table-barrier-ts":449569999358787592,"admin-job-type":0} /tidb/cdc/default/default/task/position/53ad6578-f45e-425c-908e-6058bb0cbd6b/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b0a870 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449569996527894567 MinTableBarrierTs:449569999358787592 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53ad6578-f45e-425c-908e-6058bb0cbd6b {"id":"53ad6578-f45e-425c-908e-6058bb0cbd6b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973444} /tidb/cdc/default/__cdc_meta__/capture/5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09 {"id":"5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973436} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3df5 5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3e5b 53ad6578-f45e-425c-908e-6058bb0cbd6b /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449569996527894567,"min-table-barrier-ts":449569999358787592,"admin-job-type":0} /tidb/cdc/default/default/task/position/53ad6578-f45e-425c-908e-6058bb0cbd6b/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b0a870 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449569996527894567 MinTableBarrierTs:449569999358787592 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53ad6578-f45e-425c-908e-6058bb0cbd6b {"id":"53ad6578-f45e-425c-908e-6058bb0cbd6b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973444} /tidb/cdc/default/__cdc_meta__/capture/5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09 {"id":"5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973436} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3df5 5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3e5b 53ad6578-f45e-425c-908e-6058bb0cbd6b /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449569996527894567,"min-table-barrier-ts":449569999358787592,"admin-job-type":0} /tidb/cdc/default/default/task/position/53ad6578-f45e-425c-908e-6058bb0cbd6b/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "5be7a4d4-57b9-4a3b-94a2-4e83d6ca1e09" | grep -v cluster_id | grep id =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 13:30:48.026 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:48.026 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:30:48.041 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:48.120 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:48.147 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:48.225 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] table new_ci_collation_test.t1 exists table new_ci_collation_test.t2 exists table new_ci_collation_test.t3 exists table new_ci_collation_test.t4 not exists for 1-th check, retry later You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully [2024/05/06 13:30:48.240 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:48.346 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:30:48 CST 2024] <<<<<< run test case tiflash success! >>>>>> table test.t2 not exists for 4-th check, retry later [2024/05/06 13:30:48.533 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:30:48.535 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:48.623 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:48.630 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:48.635 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:48.722 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] go: downloading github.com/sirupsen/logrus v1.8.1 go: downloading github.com/pingcap/tidb/parser v0.0.0-20211122065349-4d16aabf900b go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/pingcap/errors v0.11.5-0.20211009033009-93128226aaa3 go: downloading go.uber.org/atomic v1.9.0 go: downloading golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 "cluster-id": "default" "id": "53ad6578-f45e-425c-908e-6058bb0cbd6b", "cluster-id": "default" run task successfully [2024/05/06 13:30:48.827 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] go: downloading go.uber.org/zap v1.19.1 go: downloading github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/text v0.3.7 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 go: downloading go.uber.org/multierr v1.7.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:49.118 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:49.136 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:49.225 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:49.227 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:49.241 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCCF4415F8B57E < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 05:30:49 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact [2024/05/06 13:30:49.329 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:49.335 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:49.421 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:49.440 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] check diff successfully [Mon May 6 13:30:48 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerDontUpdateCheckpoint=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.29262928.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table new_ci_collation_test.t4 exists table new_ci_collation_test.t5 not exists for 1-th check, retry later Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... [2024/05/06 13:30:49.632 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:30:49.645 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:49.648 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:49.648 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:49.730 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:49.735 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:30:49.750 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] check diff failed 3-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/06 13:30:49.938 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:50.036 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:50.225 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:50.232 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31884e3c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3, pid:9218, start at 2024-05-06 13:30:47.074102698 +0800 CST m=+5.226364717 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:47.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:47.055 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:47.055 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31884e3c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3, pid:9218, start at 2024-05-06 13:30:47.074102698 +0800 CST m=+5.226364717 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:47.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:47.055 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:47.055 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31884fac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-c7kfw-mclzs-k9vw3, pid:9302, start at 2024-05-06 13:30:47.165961009 +0800 CST m=+5.241446413 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:47.174 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:47.147 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:47.147 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table clustered_index_test.t0 exists table clustered_index_test.t1 not exists for 1-th check, retry later [2024/05/06 13:30:50.420 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:50.433 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:50.530 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:50.539 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:30:50.547 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:50.732 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] capture_id: 53ad6578-f45e-425c-908e-6058bb0cbd6b VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318877f80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d, pid:7379, start at 2024-05-06 13:30:49.748024616 +0800 CST m=+5.327704172 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:49.755 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:49.726 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:49.726 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:50.819 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:50.820 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:50.834 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:30:50.846 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:50.849 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:30:50.919 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:51.024 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:51.045 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] wait process 2723 exit for 1-th time... table test.t2 not exists for 5-th check, retry later [2024/05/06 13:30:51.131 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 13:30:51.231 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:51.326 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:51.328 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:51.526 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] wait process 2723 exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table clustered_index_test.t1 exists table clustered_index_test.t2 not exists for 1-th check, retry later [2024/05/06 13:30:51.626 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 13:30:51.728 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] Verifying downstream PD is started... [2024/05/06 13:30:51.921 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:52.021 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:52.023 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:52 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/ebb10e78-57a7-438e-9346-3c9a7ba93b5c {UpstreamID:7365754778751316948 Namespace:default ID:ebb10e78-57a7-438e-9346-3c9a7ba93b5c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:42.123188664 +0800 CST StartTs:449569997981483009 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0023df680 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569998007697410} {CheckpointTs:449570000274980868 MinTableBarrierTs:449570000537124871 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7d898322-4653-418c-88fa-d19f95608544 {"id":"7d898322-4653-418c-88fa-d19f95608544","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973449} /tidb/cdc/default/__cdc_meta__/capture/8e391a80-79b4-476f-bb83-76e6613758cf {"id":"8e391a80-79b4-476f-bb83-76e6613758cf","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973438} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b9e5c7 8e391a80-79b4-476f-bb83-76e6613758cf /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b9e66c 7d898322-4653-418c-88fa-d19f95608544 /tidb/cdc/default/default/changefeed/info/ebb10e78-57a7-438e-9346-3c9a7ba93b5c {"upstream-id":7365754778751316948,"namespace":"default","changefeed-id":"ebb10e78-57a7-438e-9346-3c9a7ba93b5c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:42.123188664+08:00","start-ts":449569997981483009,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569998007697410} /tidb/cdc/default/default/changefeed/status/ebb10e78-57a7-438e-9346-3c9a7ba93b5c {"checkpoint-ts":449570000537124871,"min-table-barrier-ts":449570000537124871,"admin-job-type":0} /tidb/cdc/default/default/task/position/7d898322-4653-418c-88fa-d19f95608544/ebb10e78-57a7-438e-9346-3c9a7ba93b5c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/8e391a80-79b4-476f-bb83-76e6613758cf/ebb10e78-57a7-438e-9346-3c9a7ba93b5c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754778751316948 {"id":7365754778751316948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/ebb10e78-57a7-438e-9346-3c9a7ba93b5c {UpstreamID:7365754778751316948 Namespace:default ID:ebb10e78-57a7-438e-9346-3c9a7ba93b5c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:42.123188664 +0800 CST StartTs:449569997981483009 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0023df680 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569998007697410} {CheckpointTs:449570000274980868 MinTableBarrierTs:449570000537124871 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7d898322-4653-418c-88fa-d19f95608544 {"id":"7d898322-4653-418c-88fa-d19f95608544","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973449} /tidb/cdc/default/__cdc_meta__/capture/8e391a80-79b4-476f-bb83-76e6613758cf {"id":"8e391a80-79b4-476f-bb83-76e6613758cf","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973438} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b9e5c7 8e391a80-79b4-476f-bb83-76e6613758cf /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b9e66c 7d898322-4653-418c-88fa-d19f95608544 /tidb/cdc/default/default/changefeed/info/ebb10e78-57a7-438e-9346-3c9a7ba93b5c {"upstream-id":7365754778751316948,"namespace":"default","changefeed-id":"ebb10e78-57a7-438e-9346-3c9a7ba93b5c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:42.123188664+08:00","start-ts":449569997981483009,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569998007697410} /tidb/cdc/default/default/changefeed/status/ebb10e78-57a7-438e-9346-3c9a7ba93b5c {"checkpoint-ts":449570000537124871,"min-table-barrier-ts":449570000537124871,"admin-job-type":0} /tidb/cdc/default/default/task/position/7d898322-4653-418c-88fa-d19f95608544/ebb10e78-57a7-438e-9346-3c9a7ba93b5c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'failed to get info:' /tidb/cdc/default/default/task/position/8e391a80-79b4-476f-bb83-76e6613758cf/ebb10e78-57a7-438e-9346-3c9a7ba93b5c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754778751316948 {"id":7365754778751316948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/ebb10e78-57a7-438e-9346-3c9a7ba93b5c {UpstreamID:7365754778751316948 Namespace:default ID:ebb10e78-57a7-438e-9346-3c9a7ba93b5c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:42.123188664 +0800 CST StartTs:449569997981483009 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0023df680 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569998007697410} {CheckpointTs:449570000274980868 MinTableBarrierTs:449570000537124871 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7d898322-4653-418c-88fa-d19f95608544 {"id":"7d898322-4653-418c-88fa-d19f95608544","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973449} /tidb/cdc/default/__cdc_meta__/capture/8e391a80-79b4-476f-bb83-76e6613758cf {"id":"8e391a80-79b4-476f-bb83-76e6613758cf","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973438} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b9e5c7 8e391a80-79b4-476f-bb83-76e6613758cf /tidb/cdc/default/__cdc_meta__/owner/22318f4c61b9e66c 7d898322-4653-418c-88fa-d19f95608544 /tidb/cdc/default/default/changefeed/info/ebb10e78-57a7-438e-9346-3c9a7ba93b5c {"upstream-id":7365754778751316948,"namespace":"default","changefeed-id":"ebb10e78-57a7-438e-9346-3c9a7ba93b5c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:42.123188664+08:00","start-ts":449569997981483009,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569998007697410} /tidb/cdc/default/default/changefeed/status/ebb10e78-57a7-438e-9346-3c9a7ba93b5c {"checkpoint-ts":449570000537124871,"min-table-barrier-ts":449570000537124871,"admin-job-type":0} /tidb/cdc/default/default/task/position/7d898322-4653-418c-88fa-d19f95608544/ebb10e78-57a7-438e-9346-3c9a7ba93b5c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/8e391a80-79b4-476f-bb83-76e6613758cf/ebb10e78-57a7-438e-9346-3c9a7ba93b5c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754778751316948 {"id":7365754778751316948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^2$' table new_ci_collation_test.t5 exists check diff failed 1-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10677.out cli tso query --pd=http://127.0.0.1:2379 /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (2723) - No such process wait process 2723 exit for 3-th time... process 2723 already exit cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 |grep 53ad6578-f45e-425c-908e-6058bb0cbd6b -A1 | grep '"is-owner": true' check diff failed 4-th time, retry later [2024/05/06 13:30:52.119 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:52.125 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:52.131 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:30:52.134 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:52.141 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:52.237 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] 2 run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 [2024/05/06 13:30:52.444 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:52.526 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:52.540 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:52.627 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:30:52.823 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318877f80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d, pid:7379, start at 2024-05-06 13:30:49.748024616 +0800 CST m=+5.327704172 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:49.755 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:49.726 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:49.726 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318877200015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d, pid:7454, start at 2024-05-06 13:30:49.707299913 +0800 CST m=+5.229827150 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:32:49.719 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:30:49.722 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:20:49.722 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/api_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/api_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/api_v2/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/api_v2/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/api_v2/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table test.t2 not exists for 6-th check, retry later [2024/05/06 13:30:53.021 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:53.130 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 13:30:53.236 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 13:30:53.240 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:53.323 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] "is-owner": true, run task successfully test_kill_owner: pass [2024/05/06 13:30:53.424 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 13:30:53.520 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] table clustered_index_test.t2 exists check table exists success check diff failed 1-th time, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 13:30:53.634 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 13:30:53.641 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 13:30:53.650 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 13:30:53.740 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 13:30:53.836 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_gbk Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 2-th time, retry later + set +x + tso='449570000733470721 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570000733470721 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:30:54 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.1071110713.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 13:30:53.936 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 13:30:54.137 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 13:30:54.139 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] wait process cdc.test exit for 1-th time... check diff failed 5-th time, retry later [2024/05/06 13:30:54.221 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 13:30:54.257 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 13:30:54.427 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 13:30:54.429 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 13:30:54.628 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] wait process cdc.test exit for 2-th time... [2024/05/06 13:30:54.737 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] table test.t2 exists check diff successfully cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_hang_up_owner [Mon May 6 13:30:54 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.30843086.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 13:30:55.023 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 13:30:55.030 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 13:30:55.142 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 13:30:55.167 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [Mon May 6 13:30:54 CST 2024] <<<<<< START cdc server in api_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.api_v2.88408842.out server --log-file /tmp/tidb_cdc_test/api_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/api_v2/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 13:30:55.241 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs78cab445_99b2_40ff_af86_946d9fdb4113"] [2024/05/06 13:30:55.335 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 13:30:55.337 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs69ca1804_e482_4f9f_ba72_f216df21f284"] wait process cdc.test exit for 1-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:30:55.621 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs591f5cc2_b583_4051_a9a7_583484ac1194"] wait process cdc.test exit for 2-th time... check diff failed 3-th time, retry later [2024/05/06 13:30:55.843 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] check diff successfully check diff failed 2-th time, retry later [2024/05/06 13:30:56.120 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 13:30:56.129 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:56.137 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:30:56 CST 2024] <<<<<< run test case resolve_lock success! >>>>>> [2024/05/06 13:30:56.224 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:56.231 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:56.259 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:56.328 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:56.453 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] check diff failed 1-th time, retry later wait process cdc.test exit for 1-th time... [2024/05/06 13:30:56.620 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs4ff3180e_c601_4a0d_9800_6d55ca10b13a"] [2024/05/06 13:30:56.730 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:56.732 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 13:30:56.740 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:56.832 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:56.845 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:56.932 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:56.948 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] wait process cdc.test exit for 2-th time... [2024/05/06 13:30:57.228 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:57 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d442abe-2d33-4ec7-99c6-52eb4e8a180a {"id":"2d442abe-2d33-4ec7-99c6-52eb4e8a180a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973454} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ed41d4 2d442abe-2d33-4ec7-99c6-52eb4e8a180a /tidb/cdc/default/default/upstream/7365754832939853500 {"id":7365754832939853500,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d442abe-2d33-4ec7-99c6-52eb4e8a180a {"id":"2d442abe-2d33-4ec7-99c6-52eb4e8a180a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973454} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ed41d4 2d442abe-2d33-4ec7-99c6-52eb4e8a180a /tidb/cdc/default/default/upstream/7365754832939853500 {"id":7365754832939853500,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d442abe-2d33-4ec7-99c6-52eb4e8a180a {"id":"2d442abe-2d33-4ec7-99c6-52eb4e8a180a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973454} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ed41d4 2d442abe-2d33-4ec7-99c6-52eb4e8a180a /tidb/cdc/default/default/upstream/7365754832939853500 {"id":7365754832939853500,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10772.out cli changefeed create --start-ts=449570000733470721 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 13:30:57.229 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:57.325 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:57.339 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:57.368 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:57.432 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:57.464 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 5fa97685-2881-4c73-a384-4ad24b8a8d0b Info: {"upstream_id":7365754832939853500,"namespace":"default","id":"5fa97685-2881-4c73-a384-4ad24b8a8d0b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:30:57.565745922+08:00","start_ts":449570000733470721,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570000733470721,"checkpoint_ts":449570000733470721,"checkpoint_time":"2024-05-06 13:30:52.505"} PASS [2024/05/06 13:30:57.530 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:57.739 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs4bd03c2b_3742_47c6_91c0_4333e1e35de0"] wait process cdc.test exit for 3-th time... coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) [2024/05/06 13:30:57.743 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 13:30:57.755 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:57.858 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:57.939 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] cdc.test: no process found wait process cdc.test exit for 4-th time... check diff failed 3-th time, retry later ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:58 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9e3cd91e-3443-43b8-818a-85d62fecef64 {"id":"9e3cd91e-3443-43b8-818a-85d62fecef64","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973455} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61f365d9 9e3cd91e-3443-43b8-818a-85d62fecef64 /tidb/cdc/default/default/upstream/7365754845318981392 {"id":7365754845318981392,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9e3cd91e-3443-43b8-818a-85d62fecef64 {"id":"9e3cd91e-3443-43b8-818a-85d62fecef64","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973455} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61f365d9 9e3cd91e-3443-43b8-818a-85d62fecef64 /tidb/cdc/default/default/upstream/7365754845318981392 {"id":7365754845318981392,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9e3cd91e-3443-43b8-818a-85d62fecef64 {"id":"9e3cd91e-3443-43b8-818a-85d62fecef64","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973455} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61f365d9 9e3cd91e-3443-43b8-818a-85d62fecef64 /tidb/cdc/default/default/upstream/7365754845318981392 {"id":7365754845318981392,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/06 13:30:58.033 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:58.047 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:58.123 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:58.134 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] process cdc.test already exit [Mon May 6 13:30:58 CST 2024] <<<<<< run test case changefeed_pause_resume success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:30:58 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038e1950 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570001980489732 MinTableBarrierTs:449570001980489735 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570001980489735, checkpointTs: 449570001980489735, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570001980489732, checkpointTs: 449570001980489732, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570001980489735, checkpointTs: 449570001980489735, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d6730046-9ea2-4e74-9402-d326c128b671 {"id":"d6730046-9ea2-4e74-9402-d326c128b671","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973455} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3ed6 d6730046-9ea2-4e74-9402-d326c128b671 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570001980489732,"min-table-barrier-ts":449570001980489735,"admin-job-type":0} /tidb/cdc/default/default/task/position/d6730046-9ea2-4e74-9402-d326c128b671/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038e1950 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570001980489732 MinTableBarrierTs:449570001980489735 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570001980489735, checkpointTs: 449570001980489735, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570001980489732, checkpointTs: 449570001980489732, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570001980489735, checkpointTs: 449570001980489735, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d6730046-9ea2-4e74-9402-d326c128b671 {"id":"d6730046-9ea2-4e74-9402-d326c128b671","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973455} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3ed6 d6730046-9ea2-4e74-9402-d326c128b671 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570001980489732,"min-table-barrier-ts":449570001980489735,"admin-job-type":0} /tidb/cdc/default/default/task/position/d6730046-9ea2-4e74-9402-d326c128b671/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038e1950 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570001980489732 MinTableBarrierTs:449570001980489735 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570001980489735, checkpointTs: 449570001980489735, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570001980489732, checkpointTs: 449570001980489732, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570001980489735, checkpointTs: 449570001980489735, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d6730046-9ea2-4e74-9402-d326c128b671 {"id":"d6730046-9ea2-4e74-9402-d326c128b671","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973455} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3ed6 d6730046-9ea2-4e74-9402-d326c128b671 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570001980489732,"min-table-barrier-ts":449570001980489735,"admin-job-type":0} /tidb/cdc/default/default/task/position/d6730046-9ea2-4e74-9402-d326c128b671/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' check diff failed 4-th time, retry later Create changefeed successfully! ID: cf-blackhole Info: {"upstream_id":7365754845318981392,"namespace":"default","id":"cf-blackhole","sink_uri":"blackhole:","create_time":"2024-05-06T13:30:58.180062489+08:00","start_ts":449570002193874948,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570002193874948,"checkpoint_ts":449570002193874948,"checkpoint_time":"2024-05-06 13:30:58.076"} + endpoints=http://127.0.0.1:2379 + changefeed_id=cf-blackhole + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c cf-blackhole -s + info='{ "upstream_id": 7365754845318981392, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 449570002193874948, "checkpoint_time": "2024-05-06 13:30:58.076", "error": null }' + echo '{ "upstream_id": 7365754845318981392, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 449570002193874948, "checkpoint_time": "2024-05-06 13:30:58.076", "error": null }' { "upstream_id": 7365754845318981392, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 449570002193874948, "checkpoint_time": "2024-05-06 13:30:58.076", "error": null } ++ echo '{' '"upstream_id":' 7365754845318981392, '"namespace":' '"default",' '"id":' '"cf-blackhole",' '"state":' '"normal",' '"checkpoint_tso":' 449570002193874948, '"checkpoint_time":' '"2024-05-06' '13:30:58.076",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365754845318981392, '"namespace":' '"default",' '"id":' '"cf-blackhole",' '"state":' '"normal",' '"checkpoint_tso":' 449570002193874948, '"checkpoint_time":' '"2024-05-06' '13:30:58.076",' '"error":' null '}' ++ jq -r .error.message [2024/05/06 13:30:58.350 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 13:30:58.353 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:58.418 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 13:30:58.442 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + message=null + [[ ! null =~ null ]] + GO111MODULE=on + go run main.go model.go request.go cases.go go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/xdg/scram v1.0.5 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 Starting Upstream TiDB... [2024/05/06 13:30:57.100 +08:00] [INFO] [case.go:445] ["open db success"] [dsn=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:30:57.101 +08:00] [INFO] [case.go:445] ["open db success"] [dsn=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:30:57.101 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq0] [2024/05/06 13:30:57.197 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.384195ms"] [query="INSERT IGNORE INTO accounts_seq0 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/06 13:30:57.198 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq1] [2024/05/06 13:30:57.279 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.039686ms"] [query="INSERT IGNORE INTO accounts_seq1 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/06 13:30:57.279 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq2] [2024/05/06 13:30:57.362 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.259608ms"] [query="INSERT IGNORE INTO accounts_seq2 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/06 13:30:57.362 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq3] [2024/05/06 13:30:57.447 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.202185ms"] [query="INSERT IGNORE INTO accounts_seq3 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/06 13:30:57.447 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq4] [2024/05/06 13:30:57.534 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.164246ms"] [query="INSERT IGNORE INTO accounts_seq4 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/06 13:30:57.534 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts0] [2024/05/06 13:30:57.619 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.202318ms"] [query="INSERT IGNORE INTO accounts0 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/06 13:30:57.619 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts1] [2024/05/06 13:30:57.709 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.473376ms"] [query="INSERT IGNORE INTO accounts1 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/06 13:30:57.710 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts2] [2024/05/06 13:30:57.795 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.101104ms"] [query="INSERT IGNORE INTO accounts2 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/06 13:30:57.796 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts3] [2024/05/06 13:30:57.878 +08:00] [INFO] [case.go:365] ["insert 100 takes 3.98714ms"] [query="INSERT IGNORE INTO accounts3 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/06 13:30:57.879 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts4] [2024/05/06 13:30:57.961 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.046575ms"] [query="INSERT IGNORE INTO accounts4 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/06 13:30:58.520 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:58.548 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:30:58.635 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:58.725 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:30:58.737 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:30:58.754 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/YangKeao/seahash v0.0.0-20240229041150-e7bf269c3140 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later [2024/05/06 13:30:59.023 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 + set +x [2024/05/06 13:30:59.033 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:30:59.041 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 13:30:59.133 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:30:59.143 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:59.224 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:59.250 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/google/btree v1.1.2 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda [2024/05/06 13:30:59.331 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:30:59.433 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:30:59.524 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/batch_update_to_no_batch/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... "is-owner": true, run task successfully check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.33.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 [2024/05/06 13:30:59.933 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 13:30:59.938 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 13:30:59.938 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:31:00.048 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:31:00.039 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/docker/go-units v0.5.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/eapache/go-resiliency v1.4.0 [2024/05/06 13:31:00.123 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:31:00.146 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:31:00.154 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:31:00.238 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:31:00.254 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:31:00.337 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 [2024/05/06 13:31:00.636 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 13:31:00.651 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:31:00.652 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 13:31:00.743 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:31:00.837 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:31:00.929 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:31:00.939 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:31:00.945 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:31:01.027 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:31:01.030 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 [2024/05/06 13:31:01.245 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 13:31:01.248 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:31:01.250 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 13:31:01.336 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:31:01.360 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] owner pid: 3089 owner id d6730046-9ea2-4e74-9402-d326c128b671 [Mon May 6 13:31:01 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.32063208.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 check diff failed 5-th time, retry later [2024/05/06 13:31:01.518 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:31:01.522 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:31:01.536 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:31:01.557 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:31:01.628 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/gabriel-vasile/mimetype v1.4.2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:31:01.824 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 13:31:01.825 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:31:01.834 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 13:31:01.923 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 13:31:01.939 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:31:02.123 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 13:31:02.128 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:31:02.139 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:31:02.221 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] check diff failed 5-th time, retry later go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/BurntSushi/toml v1.3.2 [2024/05/06 13:31:02.241 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 13:31:02.445 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 13:31:02.446 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 13:31:02.053 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 [2024/05/06 13:31:02.532 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 13:31:02.544 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 13:31:02.731 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] start tidb cluster in /tmp/tidb_cdc_test/batch_update_to_no_batch Starting Upstream PD... go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/cockroachdb/errors v1.11.1 [2024/05/06 13:31:02.837 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 13:31:02.899 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/06 13:31:03.137 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] go: downloading github.com/cockroachdb/redact v1.1.5 [2024/05/06 13:31:03.196 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading github.com/pingcap/fn v1.0.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/tidwall/btree v1.7.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d check diff failed 6-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318924040002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5155r-hgjhp-wsj0l, pid:20788, start at 2024-05-06 13:31:00.740335179 +0800 CST m=+5.384151330 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:00.751 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:00.737 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:00.737 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318924040002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5155r-hgjhp-wsj0l, pid:20788, start at 2024-05-06 13:31:00.740335179 +0800 CST m=+5.384151330 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:00.751 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:00.737 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:00.737 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318924bc0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5155r-hgjhp-wsj0l, pid:20874, start at 2024-05-06 13:31:00.820391225 +0800 CST m=+5.393279298 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:00.830 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:00.833 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:00.833 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/httpcc v1.0.1 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > [2024/05/06 13:31:03.873 +08:00] [INFO] [main.go:812] ["testMultiDDLs take %v1m13.081395992s"] check diff failed 6-th time, retry later < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:04 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00358e000 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570003553353734 MinTableBarrierTs:449570003553353738 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68090849-ea76-4890-b20f-8838db6f2470 {"id":"68090849-ea76-4890-b20f-8838db6f2470","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973461} /tidb/cdc/default/__cdc_meta__/capture/d6730046-9ea2-4e74-9402-d326c128b671 {"id":"d6730046-9ea2-4e74-9402-d326c128b671","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973455} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3ed6 d6730046-9ea2-4e74-9402-d326c128b671 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3f27 68090849-ea76-4890-b20f-8838db6f2470 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570003553353734,"min-table-barrier-ts":449570003553353738,"admin-job-type":0} /tidb/cdc/default/default/task/position/68090849-ea76-4890-b20f-8838db6f2470/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d6730046-9ea2-4e74-9402-d326c128b671/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00358e000 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570003553353734 MinTableBarrierTs:449570003553353738 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68090849-ea76-4890-b20f-8838db6f2470 {"id":"68090849-ea76-4890-b20f-8838db6f2470","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973461} /tidb/cdc/default/__cdc_meta__/capture/d6730046-9ea2-4e74-9402-d326c128b671 {"id":"d6730046-9ea2-4e74-9402-d326c128b671","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973455} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3ed6 d6730046-9ea2-4e74-9402-d326c128b671 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3f27 68090849-ea76-4890-b20f-8838db6f2470 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570003553353734,"min-table-barrier-ts":449570003553353738,"admin-job-type":0} /tidb/cdc/default/default/task/position/68090849-ea76-4890-b20f-8838db6f2470/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d6730046-9ea2-4e74-9402-d326c128b671/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00358e000 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570003553353734 MinTableBarrierTs:449570003553353738 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68090849-ea76-4890-b20f-8838db6f2470 {"id":"68090849-ea76-4890-b20f-8838db6f2470","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973461} /tidb/cdc/default/__cdc_meta__/capture/d6730046-9ea2-4e74-9402-d326c128b671 {"id":"d6730046-9ea2-4e74-9402-d326c128b671","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973455} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3ed6 d6730046-9ea2-4e74-9402-d326c128b671 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3f27 68090849-ea76-4890-b20f-8838db6f2470 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570003553353734,"min-table-barrier-ts":449570003553353738,"admin-job-type":0} /tidb/cdc/default/default/task/position/68090849-ea76-4890-b20f-8838db6f2470/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d6730046-9ea2-4e74-9402-d326c128b671/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "d6730046-9ea2-4e74-9402-d326c128b671" | grep id [2024/05/06 13:31:04.385 +08:00] [INFO] [main.go:74] ["DefaultValue integration tests take 1m13.593282101s"] [2024/05/06 13:31:04.064 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] table mark.finish_mark_1 not exists for 1-th check, retry later go: downloading k8s.io/apimachinery v0.28.6 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/klog/v2 v2.120.1 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... "cluster-id": "default" "id": "68090849-ea76-4890-b20f-8838db6f2470", "cluster-id": "default" run task successfully go: downloading github.com/ardielle/ardielle-go v1.5.2 check diff failed 7-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 13:31:06.077 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] go: downloading github.com/jmespath/go-jmespath v0.4.0 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.22203.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) capture_id: 68090849-ea76-4890-b20f-8838db6f2470 ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '68090849-ea76-4890-b20f-8838db6f2470' 68090849-ea76-4890-b20f-8838db6f2470 run task successfully test_hang_up_owner: pass VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31895a180013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz, pid:4343, start at 2024-05-06 13:31:04.229644557 +0800 CST m=+5.436913135 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:04.240 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:04.198 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:04.198 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31895a180013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz, pid:4343, start at 2024-05-06 13:31:04.229644557 +0800 CST m=+5.436913135 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:04.240 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:04.198 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:04.198 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318959340015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz, pid:4430, start at 2024-05-06 13:31:04.187219578 +0800 CST m=+5.342051194 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:04.196 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:04.191 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:04.191 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table mark.finish_mark_1 not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31895a380019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv, pid:7868, start at 2024-05-06 13:31:04.249251322 +0800 CST m=+5.202828695 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:04.255 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:04.256 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:04.256 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31895a380019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv, pid:7868, start at 2024-05-06 13:31:04.249251322 +0800 CST m=+5.202828695 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:04.255 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:04.256 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:04.256 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31895bb40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv, pid:7943, start at 2024-05-06 13:31:04.330794797 +0800 CST m=+5.237346003 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:04.337 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:04.301 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:04.301 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... [2024/05/06 13:31:08.089 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] wait process cdc.test exit for 2-th time... check diff failed 8-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_expire_owner [Mon May 6 13:31:08 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.33643366.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_owner.server1 --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449570004595638273 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570004595638273 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:31:08 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.2225022252.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_1 not exists for 3-th check, retry later wait process cdc.test exit for 3-th time... start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:31:09 CST 2024] <<<<<< run test case new_ci_collation success! >>>>>> [Mon May 6 13:31:09 CST 2024] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.56755677.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + grep -q 'failed to get info:' + echo '' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9199.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 13:31:10.101 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/cli_with_auth/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff failed 9-th time, retry later check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_1 not exists for 4-th check, retry later + set +x + tso='449570005229502466 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570005229502466 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:31:11 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.92349236.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004294a20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570005388361735 MinTableBarrierTs:449570005650505735 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570005650505735, checkpointTs: 449570005650505735, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570005650505735, checkpointTs: 449570005650505735, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570005650505732, checkpointTs: 449570005650505732, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e72d4738-2e7c-4a11-a798-415c3433de09 {"id":"e72d4738-2e7c-4a11-a798-415c3433de09","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973468} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3f8b e72d4738-2e7c-4a11-a798-415c3433de09 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570005650505732,"min-table-barrier-ts":449570005650505735,"admin-job-type":0} /tidb/cdc/default/default/task/position/e72d4738-2e7c-4a11-a798-415c3433de09/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004294a20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570005388361735 MinTableBarrierTs:449570005650505735 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570005650505735, checkpointTs: 449570005650505735, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570005650505735, checkpointTs: 449570005650505735, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570005650505732, checkpointTs: 449570005650505732, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e72d4738-2e7c-4a11-a798-415c3433de09 {"id":"e72d4738-2e7c-4a11-a798-415c3433de09","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973468} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3f8b e72d4738-2e7c-4a11-a798-415c3433de09 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570005650505732,"min-table-barrier-ts":449570005650505735,"admin-job-type":0} /tidb/cdc/default/default/task/position/e72d4738-2e7c-4a11-a798-415c3433de09/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004294a20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570005388361735 MinTableBarrierTs:449570005650505735 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570005650505735, checkpointTs: 449570005650505735, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570005650505735, checkpointTs: 449570005650505735, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570005650505732, checkpointTs: 449570005650505732, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e72d4738-2e7c-4a11-a798-415c3433de09 {"id":"e72d4738-2e7c-4a11-a798-415c3433de09","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973468} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3f8b e72d4738-2e7c-4a11-a798-415c3433de09 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570005650505732,"min-table-barrier-ts":449570005650505735,"admin-job-type":0} /tidb/cdc/default/default/task/position/e72d4738-2e7c-4a11-a798-415c3433de09/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:12 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d312fff-6e50-4d09-a149-6830d5247daa {"id":"2d312fff-6e50-4d09-a149-6830d5247daa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973470} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c622b87d1 2d312fff-6e50-4d09-a149-6830d5247daa /tidb/cdc/default/default/upstream/7365754903424851406 {"id":7365754903424851406,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d312fff-6e50-4d09-a149-6830d5247daa {"id":"2d312fff-6e50-4d09-a149-6830d5247daa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973470} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c622b87d1 2d312fff-6e50-4d09-a149-6830d5247daa /tidb/cdc/default/default/upstream/7365754903424851406 {"id":7365754903424851406,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d312fff-6e50-4d09-a149-6830d5247daa {"id":"2d312fff-6e50-4d09-a149-6830d5247daa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973470} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c622b87d1 2d312fff-6e50-4d09-a149-6830d5247daa /tidb/cdc/default/default/upstream/7365754903424851406 {"id":7365754903424851406,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:11 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b0933bb-02d8-4926-b5cd-7b08f085f3bd {"id":"6b0933bb-02d8-4926-b5cd-7b08f085f3bd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973469} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c621e04fa 6b0933bb-02d8-4926-b5cd-7b08f085f3bd /tidb/cdc/default/default/upstream/7365754883517896943 {"id":7365754883517896943,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b0933bb-02d8-4926-b5cd-7b08f085f3bd {"id":"6b0933bb-02d8-4926-b5cd-7b08f085f3bd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973469} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c621e04fa 6b0933bb-02d8-4926-b5cd-7b08f085f3bd /tidb/cdc/default/default/upstream/7365754883517896943 {"id":7365754883517896943,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b0933bb-02d8-4926-b5cd-7b08f085f3bd {"id":"6b0933bb-02d8-4926-b5cd-7b08f085f3bd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973469} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c621e04fa 6b0933bb-02d8-4926-b5cd-7b08f085f3bd /tidb/cdc/default/default/upstream/7365754883517896943 {"id":7365754883517896943,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x wait process cdc.test exit for 3-th time... check diff failed 10-th time, retry later [2024/05/06 13:31:12.113 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table mark.finish_mark_1 not exists for 5-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:31:12 CST 2024] <<<<<< run test case owner_resign success! >>>>>> lease 22318f4c621e04fa revoked ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "is-owner": true, run task successfully start tidb cluster in /tmp/tidb_cdc_test/cli_with_auth Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table consistent_replicate_gbk.GBKTABLE not exists for 1-th check, retry later check diff failed 11-th time, retry later table capture_session_done_during_task.t exists check diff failed 1-th time, retry later [2024/05/06 13:31:14.126 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:14 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b48b2be-76fe-4699-8697-f43bf95b5d5f {"id":"6b48b2be-76fe-4699-8697-f43bf95b5d5f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973471} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c622a97d3 6b48b2be-76fe-4699-8697-f43bf95b5d5f /tidb/cdc/default/default/upstream/7365754903256776439 {"id":7365754903256776439,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b48b2be-76fe-4699-8697-f43bf95b5d5f {"id":"6b48b2be-76fe-4699-8697-f43bf95b5d5f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973471} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c622a97d3 6b48b2be-76fe-4699-8697-f43bf95b5d5f /tidb/cdc/default/default/upstream/7365754903256776439 {"id":7365754903256776439,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b48b2be-76fe-4699-8697-f43bf95b5d5f {"id":"6b48b2be-76fe-4699-8697-f43bf95b5d5f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973471} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c622a97d3 6b48b2be-76fe-4699-8697-f43bf95b5d5f /tidb/cdc/default/default/upstream/7365754903256776439 {"id":7365754903256776439,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9299.out cli changefeed create --start-ts=449570005229502466 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_1 not exists for 6-th check, retry later Create changefeed successfully! ID: a2a4bd42-97b7-4bb6-a9ab-b07ae12ea8f6 Info: {"upstream_id":7365754903256776439,"namespace":"default","id":"a2a4bd42-97b7-4bb6-a9ab-b07ae12ea8f6","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:31:14.663937073+08:00","start_ts":449570005229502466,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570005229502466,"checkpoint_ts":449570005229502466,"checkpoint_time":"2024-05-06 13:31:09.656"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... owner pid: 3369 owner id e72d4738-2e7c-4a11-a798-415c3433de09 process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v 'e72d4738-2e7c-4a11-a798-415c3433de09' /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3f8b run task successfully process status: S cdc.test cli capture list 2>&1 | grep '"is-owner": true' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3189d7f80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-st1k9-fprpv-trhhs, pid:3868, start at 2024-05-06 13:31:12.300865492 +0800 CST m=+5.530371633 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:12.309 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:12.304 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:12.304 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3189d7f80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-st1k9-fprpv-trhhs, pid:3868, start at 2024-05-06 13:31:12.300865492 +0800 CST m=+5.530371633 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:12.309 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:12.304 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:12.304 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3189d87c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-st1k9-fprpv-trhhs, pid:3953, start at 2024-05-06 13:31:12.329702678 +0800 CST m=+5.494670496 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:12.341 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:12.337 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:12.337 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3189eec80011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-r8vbf-lj78v-9fw4x, pid:3912, start at 2024-05-06 13:31:13.732746755 +0800 CST m=+5.139500988 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:13.739 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:13.714 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:13.714 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3189eec80011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-r8vbf-lj78v-9fw4x, pid:3912, start at 2024-05-06 13:31:13.732746755 +0800 CST m=+5.139500988 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:13.739 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:13.714 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:13.714 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3189f10c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-r8vbf-lj78v-9fw4x, pid:3993, start at 2024-05-06 13:31:13.871267438 +0800 CST m=+5.226052554 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:13.877 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:13.859 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:13.859 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } build mysql_test successfully run all mysql test cases time="2024-05-06T13:31:12+08:00" level=info msg="running tests: [alter_table bigint composite_index concurrent_ddl date_formats drop mysql_replace]" time="2024-05-06T13:31:12+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc000298000 {{} {} 0} {0 0} [0xc000314000] map[] 0 1 0xc00029c000 false map[0xc000314000:map[0xc000314000:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" "is-owner": true, run task successfully test_expire_owner pass + set +x check diff failed 2-th time, retry later [2024/05/06 13:31:16.137 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] table consistent_replicate_gbk.GBKTABLE exists table consistent_replicate_gbk.check1 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... check diff failed 12-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 13:31:17 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.52665268.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.t exists table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... table mark.finish_mark_1 not exists for 7-th check, retry later table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_owner_cleanup_stale_tasks [Mon May 6 13:31:17 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.35543556.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 13:31:18.150 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] table consistent_replicate_gbk.check1 not exists for 2-th check, retry later check diff failed 13-th time, retry later check diff failed 3-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_1 not exists for 8-th check, retry later [Mon May 6 13:31:19 CST 2024] <<<<<< START cdc server in batch_update_to_no_batch case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.52515253.out server --log-file /tmp/tidb_cdc_test/batch_update_to_no_batch/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_update_to_no_batch/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.finish_mark not exists for 2-th check, retry later [2024/05/06 13:31:20.163 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] [2024/05/06 13:31:20.177 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark] [ts=449570007740579845] [2024/05/06 13:31:20.177 +08:00] [INFO] [case.go:498] ["all tables synced"] [endTs=449570007740579845] Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_gbk.check1 exists check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:20 GMT < Content-Length: 853 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/23836227-08db-430c-9af2-3cbce42d79dd {"id":"23836227-08db-430c-9af2-3cbce42d79dd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973477} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c6250733a 23836227-08db-430c-9af2-3cbce42d79dd /tidb/cdc/default/default/upstream/7365754937969433047 {"id":7365754937969433047,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/23836227-08db-430c-9af2-3cbce42d79dd {"id":"23836227-08db-430c-9af2-3cbce42d79dd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973477} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c6250733a 23836227-08db-430c-9af2-3cbce42d79dd /tidb/cdc/default/default/upstream/7365754937969433047 {"id":7365754937969433047,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/23836227-08db-430c-9af2-3cbce42d79dd {"id":"23836227-08db-430c-9af2-3cbce42d79dd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973477} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c6250733a 23836227-08db-430c-9af2-3cbce42d79dd /tidb/cdc/default/default/upstream/7365754937969433047 {"id":7365754937969433047,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 6515b346-33e0-4902-98ef-38b3b0be3ef0 Info: {"upstream_id":7365754937969433047,"namespace":"default","id":"6515b346-33e0-4902-98ef-38b3b0be3ef0","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:31:20.371399655+08:00","start_ts":449570008007704578,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570008007704578,"checkpoint_ts":449570008007704578,"checkpoint_time":"2024-05-06 13:31:20.254"} check diff failed 14-th time, retry later check diff successfully check diff failed 1-th time, retry later wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318a4d040016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd, pid:23801, start at 2024-05-06 13:31:19.775874052 +0800 CST m=+5.182552878 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:19.785 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:19.745 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:19.745 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318a4d040016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd, pid:23801, start at 2024-05-06 13:31:19.775874052 +0800 CST m=+5.182552878 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:19.785 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:19.745 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:19.745 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318a4f400009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd, pid:23874, start at 2024-05-06 13:31:19.897109145 +0800 CST m=+5.251588843 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:19.904 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:19.888 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:19.888 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/move_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/move_table/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/move_table/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003013c20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570007760502786 MinTableBarrierTs:449570008022908931 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570008009801733, checkpointTs: 449570008009801733, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570008022908931, checkpointTs: 449570008022908931, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570008022908931, checkpointTs: 449570008022908931, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3056f829-8a4b-4430-9103-e25d672dbc67 {"id":"3056f829-8a4b-4430-9103-e25d672dbc67","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973477} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3fff 3056f829-8a4b-4430-9103-e25d672dbc67 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570008009801733,"min-table-barrier-ts":449570008022908931,"admin-job-type":0} /tidb/cdc/default/default/task/position/3056f829-8a4b-4430-9103-e25d672dbc67/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003013c20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570007760502786 MinTableBarrierTs:449570008022908931 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570008009801733, checkpointTs: 449570008009801733, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570008022908931, checkpointTs: 449570008022908931, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570008022908931, checkpointTs: 449570008022908931, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3056f829-8a4b-4430-9103-e25d672dbc67 {"id":"3056f829-8a4b-4430-9103-e25d672dbc67","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973477} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3fff 3056f829-8a4b-4430-9103-e25d672dbc67 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570008009801733,"min-table-barrier-ts":449570008022908931,"admin-job-type":0} /tidb/cdc/default/default/task/position/3056f829-8a4b-4430-9103-e25d672dbc67/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003013c20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570007760502786 MinTableBarrierTs:449570008022908931 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570008009801733, checkpointTs: 449570008009801733, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570008022908931, checkpointTs: 449570008022908931, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570008022908931, checkpointTs: 449570008022908931, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3056f829-8a4b-4430-9103-e25d672dbc67 {"id":"3056f829-8a4b-4430-9103-e25d672dbc67","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973477} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3fff 3056f829-8a4b-4430-9103-e25d672dbc67 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570008009801733,"min-table-barrier-ts":449570008022908931,"admin-job-type":0} /tidb/cdc/default/default/task/position/3056f829-8a4b-4430-9103-e25d672dbc67/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' wait process cdc.test exit for 2-th time... table mark.finish_mark_1 not exists for 9-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 3-th time... split_and_random_merge scale: 20 table partition_table.finish_mark not exists for 3-th check, retry later "is-owner": true, run task successfully cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:31:22 CST 2024] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.58825884.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_sequence.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:22 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ab87b665-f703-4916-be95-ca7dbcec5c66 {"id":"ab87b665-f703-4916-be95-ca7dbcec5c66","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973479} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c6251afd3 ab87b665-f703-4916-be95-ca7dbcec5c66 /tidb/cdc/default/default/upstream/7365754943604714677 {"id":7365754943604714677,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ab87b665-f703-4916-be95-ca7dbcec5c66 {"id":"ab87b665-f703-4916-be95-ca7dbcec5c66","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973479} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c6251afd3 ab87b665-f703-4916-be95-ca7dbcec5c66 /tidb/cdc/default/default/upstream/7365754943604714677 {"id":7365754943604714677,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ab87b665-f703-4916-be95-ca7dbcec5c66 {"id":"ab87b665-f703-4916-be95-ca7dbcec5c66","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973479} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c6251afd3 ab87b665-f703-4916-be95-ca7dbcec5c66 /tidb/cdc/default/default/upstream/7365754943604714677 {"id":7365754943604714677,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5313.out cli changefeed create '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=true' -c test Create changefeed successfully! ID: test Info: {"upstream_id":7365754943604714677,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=true","create_time":"2024-05-06T13:31:22.690317203+08:00","start_ts":449570008613257220,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570008613257220,"checkpoint_ts":449570008613257220,"checkpoint_time":"2024-05-06 13:31:22.564"} PASS =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... coverage: 2.4% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_1 exists table mark.finish_mark_2 not exists for 1-th check, retry later check diff failed 15-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.25119.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 13:31:22.937 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark1000] [2024/05/06 13:31:22.937 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark1000] ____________________________________ *************************** 1. row *************************** primary_ts: 449569942083731457 secondary_ts: 449569944794824717 *************************** 2. row *************************** primary_ts: 449569949948051456 secondary_ts: 449569950483349505 *************************** 3. row *************************** primary_ts: 449569957812371456 secondary_ts: 449569958242811905 *************************** 4. row *************************** primary_ts: 449569965676691456 secondary_ts: 449569965976059905 *************************** 5. row *************************** primary_ts: 449569973541011456 secondary_ts: 449569973853749255 *************************** 6. row *************************** primary_ts: 449569981405331456 secondary_ts: 449569982242357253 *************************** 7. row *************************** primary_ts: 449569989269651456 secondary_ts: 449569989582127111 *************************** 8. row *************************** primary_ts: 449569997133971456 secondary_ts: 449569997446447111 *************************** 9. row *************************** primary_ts: 449570004998291456 secondary_ts: 449570005337243650 skip invalid syncpoint primary_ts: 449569942083731457, first_ddl_ts: 449569945517817888 check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:31:21 CST 2024] <<<<<< run test case syncpoint success! >>>>>> table partition_table.finish_mark not exists for 4-th check, retry later wait process cdc.test exit for 2-th time... owner pid: 3559 owner id 3056f829-8a4b-4430-9103-e25d672dbc67 [Mon May 6 13:31:23 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.36823684.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:31:24 CST 2024] <<<<<< run test case capture_session_done_during_task success! >>>>>> + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_sequence.finish_mark not exists for 2-th check, retry later + set +x + tso='449570008778670081 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570008778670081 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table mark.finish_mark_2 exists table mark.finish_mark_3 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 16-th time, retry later [2024/05/06 13:31:24.938 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark1000] [2024/05/06 13:31:25.133 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark2000] [2024/05/06 13:31:25.133 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark2000] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:25 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/d3fbc8a8-242e-47fe-a236-245f7cb0cd1c {UpstreamID:7365754903424851406 Namespace:default ID:d3fbc8a8-242e-47fe-a236-245f7cb0cd1c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:31:12.373601724 +0800 CST StartTs:449570005909241861 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025c9680 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570005935194119} {CheckpointTs:449570008962957314 MinTableBarrierTs:449570009225101315 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570008962957314, checkpointTs: 449570008962957314, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570008962957314, checkpointTs: 449570008962957314, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/da00ad84-f1fe-4235-8a23-82baeffe4264 {"id":"da00ad84-f1fe-4235-8a23-82baeffe4264","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973482} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c622b8883 da00ad84-f1fe-4235-8a23-82baeffe4264 /tidb/cdc/default/default/changefeed/info/d3fbc8a8-242e-47fe-a236-245f7cb0cd1c {"upstream-id":7365754903424851406,"namespace":"default","changefeed-id":"d3fbc8a8-242e-47fe-a236-245f7cb0cd1c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:31:12.373601724+08:00","start-ts":449570005909241861,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570005935194119} /tidb/cdc/default/default/changefeed/status/d3fbc8a8-242e-47fe-a236-245f7cb0cd1c {"checkpoint-ts":449570008962957314,"min-table-barrier-ts":449570009225101315,"admin-job-type":0} /tidb/cdc/default/default/task/position/da00ad84-f1fe-4235-8a23-82baeffe4264/d3fbc8a8-242e-47fe-a236-245f7cb0cd1c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754903424851406 {"id":7365754903424851406,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/d3fbc8a8-242e-47fe-a236-245f7cb0cd1c {UpstreamID:7365754903424851406 Namespace:default ID:d3fbc8a8-242e-47fe-a236-245f7cb0cd1c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:31:12.373601724 +0800 CST StartTs:449570005909241861 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025c9680 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570005935194119} {CheckpointTs:449570008962957314 MinTableBarrierTs:449570009225101315 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570008962957314, checkpointTs: 449570008962957314, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570008962957314, checkpointTs: 449570008962957314, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/da00ad84-f1fe-4235-8a23-82baeffe4264 {"id":"da00ad84-f1fe-4235-8a23-82baeffe4264","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973482} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c622b8883 da00ad84-f1fe-4235-8a23-82baeffe4264 /tidb/cdc/default/default/changefeed/info/d3fbc8a8-242e-47fe-a236-245f7cb0cd1c {"upstream-id":7365754903424851406,"namespace":"default","changefeed-id":"d3fbc8a8-242e-47fe-a236-245f7cb0cd1c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:31:12.373601724+08:00","start-ts":449570005909241861,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570005935194119} /tidb/cdc/default/default/changefeed/status/d3fbc8a8-242e-47fe-a236-245f7cb0cd1c {"checkpoint-ts":449570008962957314,"min-table-barrier-ts":449570009225101315,"admin-job-type":0} /tidb/cdc/default/default/task/position/da00ad84-f1fe-4235-8a23-82baeffe4264/d3fbc8a8-242e-47fe-a236-245f7cb0cd1c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754903424851406 {"id":7365754903424851406,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/d3fbc8a8-242e-47fe-a236-245f7cb0cd1c {UpstreamID:7365754903424851406 Namespace:default ID:d3fbc8a8-242e-47fe-a236-245f7cb0cd1c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:31:12.373601724 +0800 CST StartTs:449570005909241861 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025c9680 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570005935194119} {CheckpointTs:449570008962957314 MinTableBarrierTs:449570009225101315 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570008962957314, checkpointTs: 449570008962957314, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570008962957314, checkpointTs: 449570008962957314, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/da00ad84-f1fe-4235-8a23-82baeffe4264 {"id":"da00ad84-f1fe-4235-8a23-82baeffe4264","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973482} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c622b8883 da00ad84-f1fe-4235-8a23-82baeffe4264 /tidb/cdc/default/default/changefeed/info/d3fbc8a8-242e-47fe-a236-245f7cb0cd1c {"upstream-id":7365754903424851406,"namespace":"default","changefeed-id":"d3fbc8a8-242e-47fe-a236-245f7cb0cd1c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:31:12.373601724+08:00","start-ts":449570005909241861,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570005935194119} /tidb/cdc/default/default/changefeed/status/d3fbc8a8-242e-47fe-a236-245f7cb0cd1c {"checkpoint-ts":449570008962957314,"min-table-barrier-ts":449570009225101315,"admin-job-type":0} /tidb/cdc/default/default/task/position/da00ad84-f1fe-4235-8a23-82baeffe4264/d3fbc8a8-242e-47fe-a236-245f7cb0cd1c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"+ grep -q 'etcd info' warning":null} /tidb/cdc/default/default/upstream/7365754903424851406 {"id":7365754903424851406,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table partition_table.finish_mark not exists for 5-th check, retry later [2024/05/06 13:31:24.136 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 20"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318aa3f40002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd, pid:10896, start at 2024-05-06 13:31:25.3108239 +0800 CST m=+5.241794800 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:25.317 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:25.309 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:25.309 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318aa3f40002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd, pid:10896, start at 2024-05-06 13:31:25.3108239 +0800 CST m=+5.241794800 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:25.317 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:25.309 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:25.309 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318aa5600014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd, pid:10981, start at 2024-05-06 13:31:25.424738926 +0800 CST m=+5.291976610 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:25.432 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:25.400 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:25.400 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_sequence.finish_mark not exists for 3-th check, retry later table mark.finish_mark_3 not exists for 2-th check, retry later check diff failed 17-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' [2024/05/06 13:31:26.949 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark1000] > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 [2024/05/06 13:31:27.041 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark1000] [ts=449570009309773829] > Host: 127.0.0.1:8301 > Accept: */* > [2024/05/06 13:31:27.041 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark1000] < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:26 GMT < Content-Type: text/plain; charset=utf-8 [2024/05/06 13:31:27.041 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570009309773829] < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' [2024/05/06 13:31:27.044 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 [2024/05/06 13:31:27.128 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:31:27.133 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:27.133 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark2000] [2024/05/06 13:31:27.138 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001420fc0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570009333628938 MinTableBarrierTs:449570009595510793 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3056f829-8a4b-4430-9103-e25d672dbc67 {"id":"3056f829-8a4b-4430-9103-e25d672dbc67","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973477} /tidb/cdc/default/__cdc_meta__/capture/588618e0-e261-4ce1-9645-d979067dbc13 {"id":"588618e0-e261-4ce1-9645-d979067dbc13","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973483} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3fff 3056f829-8a4b-4430-9103-e25d672dbc67 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac404b 588618e0-e261-4ce1-9645-d979067dbc13 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570009333628938,"min-table-barrier-ts":449570009595510793,"admin-job-type":0} /tidb/cdc/default/default/task/position/3056f829-8a4b-4430-9103-e25d672dbc67/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/588618e0-e261-4ce1-9645-d979067dbc13/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001420fc0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570009333628938 MinTableBarrierTs:449570009595510793 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3056f829-8a4b-4430-9103-e25d672dbc67 {"id":"3056f829-8a4b-4430-9103-e25d672dbc67","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973477} /tidb/cdc/default/__cdc_meta__/capture/588618e0-e261-4ce1-9645-d979067dbc13 {"id":"588618e0-e261-4ce1-9645-d979067dbc13","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973483} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3fff 3056f829-8a4b-4430-9103-e25d672dbc67 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac404b 588618e0-e261-4ce1-9645-d979067dbc13 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570009333628938,"min-table-barrier-ts":449570009595510793,"admin-job-type":0} /tidb/cdc/default/default/task/position/3056f829-8a4b-4430-9103-e25d672dbc67/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/588618e0-e261-4ce1-9645-d979067dbc13/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001420fc0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570009333628938 MinTableBarrierTs:449570009595510793 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3056f829-8a4b-4430-9103-e25d672dbc67 {"id":"3056f829-8a4b-4430-9103-e25d672dbc67","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973477} /tidb/cdc/default/__cdc_meta__/capture/588618e0-e261-4ce1-9645-d979067dbc13 {"id":"588618e0-e261-4ce1-9645-d979067dbc13","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973483} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3fff 3056f829-8a4b-4430-9103-e25d672dbc67 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac404b 588618e0-e261-4ce1-9645-d979067dbc13 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570009333628938,"min-table-barrier-ts":449570009595510793,"admin-job-type":0} /tidb/cdc/default/default/task/position/3056f829-8a4b-4430-9103-e25d672dbc67/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /+ grep -q 'etcd info' tidb/cdc/default/default/task/position/588618e0-e261-4ce1-9645-d979067dbc13/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "3056f829-8a4b-4430-9103-e25d672dbc67" | grep id ***************** properties ***************** "threadcount"="10" "readallfields"="true" "updateproportion"="0" "insertproportion"="0" "requestdistribution"="uniform" "operationcount"="0" "recordcount"="10000" "workload"="core" "scanproportion"="0" "mysql.port"="4000" "dotransactions"="false" "mysql.host"="127.0.0.1" "readproportion"="0" "mysql.db"="move_table" "mysql.user"="root" ********************************************** table partition_table.finish_mark not exists for 6-th check, retry later "cluster-id": "default" "id": "588618e0-e261-4ce1-9645-d979067dbc13", "cluster-id": "default" run task successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... *************************** 1. row *************************** count(distinct region_id): 1 table ddl_sequence.finish_mark not exists for 4-th check, retry later [2024/05/06 13:31:28.432 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark3000] [2024/05/06 13:31:28.432 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark3000] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12238.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 13:31:29.150 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark2000] [2024/05/06 13:31:29.139 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 29"] Run finished, takes 2.422364634s INSERT - Takes(s): 2.4, Count: 10000, OPS: 4138.1, Avg(us): 2375, Min(us): 774, Max(us): 291471, 95th(us): 2000, 99th(us): 41000 [Mon May 6 13:31:29 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2520625208.out server --log-file /tmp/tidb_cdc_test/move_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 3-th check, retry later [2024/05/06 13:31:29.238 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark2000] [ts=449570010191626245] [2024/05/06 13:31:29.238 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark2000] [2024/05/06 13:31:29.238 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570010191626245] [2024/05/06 13:31:29.324 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:29.328 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:31:29.333 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:29.338 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] check diff failed 18-th time, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5364.out cli changefeed pause -c test capture_id: 588618e0-e261-4ce1-9645-d979067dbc13 table partition_table.finish_mark not exists for 7-th check, retry later Verifying downstream PD is started... PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x + tso='449570010237239297 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570010237239297 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table ddl_sequence.finish_mark not exists for 5-th check, retry later [Mon May 6 13:31:30 CST 2024] <<<<<< START cdc server in cli_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.1228512287.out server --log-file /tmp/tidb_cdc_test/cli_with_auth/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_with_auth/cdc_data --cluster-id default --config /tmp/tidb_cdc_test/cli_with_auth/server.toml + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 13:31:30.432 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark3000] table mark.finish_mark_3 not exists for 4-th check, retry later check diff failed 19-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5401.out cli changefeed update -c test '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=false' --no-confirm [2024/05/06 13:31:31.634 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark4000] [2024/05/06 13:31:31.634 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark4000] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:31:30 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> table partition_table.finish_mark not exists for 8-th check, retry later Diff of changefeed config: {Type:update Path:[SinkURI] From:mysql://root@127.0.0.1:3306/?batch-dml-enable=true To:mysql://root@127.0.0.1:3306/?batch-dml-enable=false} Update changefeed config successfully! ID: test Info: {"upstream_id":7365754943604714677,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=false","create_time":"2024-05-06T13:31:22.690317203+08:00","start_ts":449570008613257220,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":0,"checkpoint_ts":449570009347260444,"checkpoint_time":"2024-05-06 13:31:25.364"} PASS check_redo_resolved_ts d3fbc8a8-242e-47fe-a236-245f7cb0cd1c 449570010798227463 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/d3fbc8a8-242e-47fe-a236-245f7cb0cd1c/meta global resolved ts 449570009329958934 not forward to 449570010798227463 run task failed 1-th time, retry later coverage: 2.1% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:32 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811 {"id":"7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973489} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c626b3344 7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811 /tidb/cdc/default/default/upstream/7365754973417589185 {"id":7365754973417589185,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811 {"id":"7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973489} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c626b3344 7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811 /tidb/cdc/default/default/upstream/7365754973417589185 {"id":7365754973417589185,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811 {"id":"7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973489} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c626b3344 7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811 /tidb/cdc/default/default/upstream/7365754973417589185 {"id":7365754973417589185,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.25267.out cli changefeed create --start-ts=449570008778670081 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' split_and_random_merge scale: 40 [2024/05/06 13:31:32.521 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark3000] [2024/05/06 13:31:32.538 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark3000] [ts=449570011118567429] [2024/05/06 13:31:32.538 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark3000] [2024/05/06 13:31:32.538 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570011118567429] Create changefeed successfully! ID: 9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 Info: {"upstream_id":7365754973417589185,"namespace":"default","id":"9cbe3e2c-4ad8-4840-8911-e0dbef1c4359","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T13:31:32.691518234+08:00","start_ts":449570008778670081,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570008778670081,"checkpoint_ts":449570008778670081,"checkpoint_time":"2024-05-06 13:31:23.195"} PASS [2024/05/06 13:31:32.623 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:32.627 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:31:32.631 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:32.638 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table ddl_sequence.finish_mark not exists for 6-th check, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... check_redo_resolved_ts d3fbc8a8-242e-47fe-a236-245f7cb0cd1c 449570010798227463 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/d3fbc8a8-242e-47fe-a236-245f7cb0cd1c/meta table mark.finish_mark_3 not exists for 5-th check, retry later run task successfully 0 [Mon May 6 13:31:32 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.38253827.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server3.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server3 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 20-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:33 GMT < Content-Length: 853 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd3ed6c4-1370-40ab-9668-a3221fe2041c {"id":"bd3ed6c4-1370-40ab-9668-a3221fe2041c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973490} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c62827a46 bd3ed6c4-1370-40ab-9668-a3221fe2041c /tidb/cdc/default/default/upstream/7365754993661904829 {"id":7365754993661904829,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd3ed6c4-1370-40ab-9668-a3221fe2041c {"id":"bd3ed6c4-1370-40ab-9668-a3221fe2041c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973490} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c62827a46 bd3ed6c4-1370-40ab-9668-a3221fe2041c /tidb/cdc/default/default/upstream/7365754993661904829 {"id":7365754993661904829,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd3ed6c4-1370-40ab-9668-a3221fe2041c {"id":"bd3ed6c4-1370-40ab-9668-a3221fe2041c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973490} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c62827a46 bd3ed6c4-1370-40ab-9668-a3221fe2041c /tidb/cdc/default/default/upstream/7365754993661904829 {"id":7365754993661904829,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12352.out cli changefeed create --start-ts=449570010237239297 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name ./t/alter_table.test: ok! 64 test cases passed, take time 17.979664987 s time="2024-05-06T13:31:30+08:00" level=info msg="run test [alter_table] ok" time="2024-05-06T13:31:30+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0000e8308 {{} {} 0} {0 0} [0xc0003145a0] map[] 0 1 0xc000036a80 false map[0xc0003145a0:map[0xc0003145a0:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5445.out cli changefeed resume -c test [2024/05/06 13:31:33.634 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] table partition_table.finish_mark not exists for 9-th check, retry later wait process cdc.test exit for 1-th time... [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365754993661904829,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:31:33.949926786+08:00","start_ts":449570010237239297,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570010237239297,"checkpoint_ts":449570010237239297,"checkpoint_time":"2024-05-06 13:31:28.759"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x [Mon May 6 13:31:34 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2530125303.out server --log-file /tmp/tidb_cdc_test/move_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + grep -q 'failed to get info:' + echo '' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [2024/05/06 13:31:34.634 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark5000] [2024/05/06 13:31:34.634 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark5000] table ddl_sequence.finish_mark exists check diff successfully [2024/05/06 13:31:34.741 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 13:31:34.741 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2024/05/06 13:31:34.742 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=:6060] [2024/05/06 13:31:34.743 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/06 13:31:34.745 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 13:31:34.745 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449570009329958934] [resolvedTs=449570011060109315] [2024/05/06 13:31:34.746 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/06 13:31:34.746 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/06 13:31:34.746 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 13:31:34.746 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 13:31:34.746 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2024/05/06 13:31:34.750 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=4.215235ms] [2024/05/06 13:31:34.751 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 13:31:34.751 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=5.783265ms] [2024/05/06 13:31:34.753 +08:00] [INFO] [mysql.go:197] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 13:31:34.754 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:31:34.754 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:31:34.754 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:31:34.754 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:31:34.754 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:31:34.754 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:31:34.754 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:31:34.754 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:31:34.754 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:31:34.754 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:31:34.754 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:31:34.754 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:31:34.754 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:31:34.754 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:31:34.754 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:31:34.754 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:31:34.755 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 13:31:34.755 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 13:31:34.755 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 13:31:34.755 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 13:31:34.755 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 13:31:34.755 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 13:31:34.755 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 13:31:34.755 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 13:31:34.755 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 13:31:34.755 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 13:31:34.755 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 13:31:34.755 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 13:31:34.755 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 13:31:34.755 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 13:31:34.755 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 13:31:34.755 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 13:31:34.755 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 13:31:34.755 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 13:31:34.755 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2024/05/06 13:31:34.758 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 13:31:34.760 +08:00] [INFO] [mysql_ddl_sink.go:107] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/06 13:31:34.760 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449570009329958929,\"CommitTs\":449570009329958934,\"Query\":\"create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_gbk\",\"Table\":\"GBKTABLE2\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/06 13:31:34.761 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE"] [commitTs=449570009329958934] [namespace=] [changefeed=] [2024/05/06 13:31:34.762 +08:00] [INFO] [mysql_ddl_sink.go:127] ["Execute DDL failed, but error can be ignored"] [startTs=449570009329958929] [ddl="create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE"] [namespace=] [changefeed=] [error="Error 1050 (42S01): Table 'consistent_replicate_gbk.GBKTABLE2' already exists"] table mark.finish_mark_3 not exists for 6-th check, retry later wait process cdc.test exit for 1-th time... [2024/05/06 13:31:34.962 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449570011060109315] [2024/05/06 13:31:34.962 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449570011060109315] [2024/05/06 13:31:35.062 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=1] [appliedDDLCount=1] [currentCheckpoint=449570011060109315] [2024/05/06 13:31:35.062 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 13:31:35.062 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 13:31:35.062 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 13:31:35.062 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/06 13:31:35.062 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 13:31:35.062 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/06 13:31:35.063 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 13:31:35.063 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/06 13:31:35.062 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/06 13:31:35.062 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 13:31:35.062 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 13:31:35.062 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 13:31:35.062 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/06 13:31:35.062 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 13:31:35.063 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 13:31:35.063 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] Apply redo log successfully check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table test.simple exists table test.`simple-dash` not exists for 1-th check, retry later [Mon May 6 13:31:35 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT wait process cdc.test exit for 2-th time... table partition_table.finish_mark not exists for 10-th check, retry later [2024/05/06 13:31:35.650 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] [2024/05/06 13:31:35.666 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark4000] [ts=449570011118567429] [2024/05/06 13:31:35.666 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark4000] [2024/05/06 13:31:35.666 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570011118567429] [2024/05/06 13:31:35.668 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:35 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc000e81ef0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570010120060939 MinTableBarrierTs:449570010381942793 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3056f829-8a4b-4430-9103-e25d672dbc67 {"id":"3056f829-8a4b-4430-9103-e25d672dbc67","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973477} /tidb/cdc/default/__cdc_meta__/capture/525aca4a-69de-4293-80fe-969a425e68cb {"id":"525aca4a-69de-4293-80fe-969a425e68cb","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973492} /tidb/cdc/default/__cdc_meta__/capture/588618e0-e261-4ce1-9645-d979067dbc13 {"id":"588618e0-e261-4ce1-9645-d979067dbc13","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973483} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3fff 3056f829-8a4b-4430-9103-e25d672dbc67 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac404b 588618e0-e261-4ce1-9645-d979067dbc13 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac40c3 525aca4a-69de-4293-80fe-969a425e68cb /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570010120060939,"min-table-barrier-ts":449570010381942793,"admin-job-type":0} /tidb/cdc/default/default/task/position/3056f829-8a4b-4430-9103-e25d672dbc67/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/525aca4a-69de-4293-80fe-969a425e68cb/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/588618e0-e261-4ce1-9645-d979067dbc13/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc000e81ef0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570010120060939 MinTableBarrierTs:449570010381942793 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3056f829-8a4b-4430-9103-e25d672dbc67 {"id":"3056f829-8a4b-4430-9103-e25d672dbc67","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973477} /tidb/cdc/default/__cdc_meta__/capture/525aca4a-69de-4293-80fe-969a425e68cb {"id":"525aca4a-69de-4293-80fe-969a425e68cb","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973492} /tidb/cdc/default/__cdc_meta__/capture/588618e0-e261-4ce1-9645-d979067dbc13 {"id":"588618e0-e261-4ce1-9645-d979067dbc13","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973483} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3fff 3056f829-8a4b-4430-9103-e25d672dbc67 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac404b 588618e0-e261-4ce1-9645-d979067dbc13 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac40c3 525aca4a-69de-4293-80fe-969a425e68cb /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570010120060939,"min-table-barrier-ts":449570010381942793,"admin-job-type":0} /tidb/cdc/default/default/task/position/3056f829-8a4b-4430-9103-e25d672dbc67/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/525aca4a-69de-4293-80fe-969a425e68cb/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/588618e0-e261-4ce1-9645-d979067dbc13/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc000e81ef0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570010120060939 MinTableBarrierTs:449570010381942793 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3056f829-8a4b-4430-9103-e25d672dbc67 {"id":"3056f829-8a4b-4430-9103-e25d672dbc67","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973477} /tidb/cdc/default/__cdc_meta__/capture/525aca4a-69de-4293-80fe-969a425e68cb {"id":"525aca4a-69de-4293-80fe-969a425e68cb","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973492} /tidb/cdc/default/__cdc_meta__/capture/588618e0-e261-4ce1-9645-d979067dbc13 {"id":"588618e0-e261-4ce1-9645-d979067dbc13","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973483} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac3fff 3056f829-8a4b-4430-9103-e25d672dbc67 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac404b 588618e0-e261-4ce1-9645-d979067dbc13 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac40c3 525aca4a-69de-4293-80fe-969a425e68cb /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570010120060939,"min-table-barrier-ts":449570010381942793,"admin-job-type":0} /tidb/cdc/default/default/task/position/3056f829-8a4b-4430-9103-e25d672dbc67/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/525aca4a-69de-4293-80fe-969a425e68cb/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/588618e0-e261-4ce1-9645-d979067dbc13/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' check diff failed 21-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:31:35.724 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:31:35.728 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:35.734 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] run task failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:31:36 CST 2024] <<<<<< run test case ddl_sequence success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 13:31:36.634 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] table mark.finish_mark_3 not exists for 7-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318b4f7c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-63kf5-dnt52-vw515, pid:4109, start at 2024-05-06 13:31:36.33006562 +0800 CST m=+5.354328138 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:36.338 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:36.337 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:36.337 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 11-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:37 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {UpstreamID:7365754973417589185 Namespace:default ID:9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:31:32.691518234 +0800 CST StartTs:449570008778670081 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031601b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570011255930893} {CheckpointTs:449570009434030104 MinTableBarrierTs:449570012304506889 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3398e8ae-814f-4a43-9659-7de7839827c5 {"id":"3398e8ae-814f-4a43-9659-7de7839827c5","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973494} /tidb/cdc/default/__cdc_meta__/capture/7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811 {"id":"7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973489} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c626b3344 7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811 /tidb/cdc/default/__cdc_meta__/owner/22318f4c626b339c 3398e8ae-814f-4a43-9659-7de7839827c5 /tidb/cdc/default/default/changefeed/info/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"upstream-id":7365754973417589185,"namespace":"default","changefeed-id":"9cbe3e2c-4ad8-4840-8911-e0dbef1c4359","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:31:32.691518234+08:00","start-ts":449570008778670081,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570011255930893} /tidb/cdc/default/default/changefeed/status/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":449570009434030104,"min-table-barrier-ts":449570012304506889,"admin-job-type":0} /tidb/cdc/default/default/task/position/3398e8ae-814f-4a43-9659-7de7839827c5/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754973417589185 {"id":7365754973417589185,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {UpstreamID:7365754973417589185 Namespace:default ID:9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:31:32.691518234 +0800 CST StartTs:449570008778670081 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031601b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570011255930893} {CheckpointTs:449570009434030104 MinTableBarrierTs:449570012304506889 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3398e8ae-814f-4a43-9659-7de7839827c5 {"id":"3398e8ae-814f-4a43-9659-7de7839827c5","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973494} /tidb/cdc/default/__cdc_meta__/capture/7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811 {"id":"7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973489} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c626b3344 7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811 /tidb/cdc/default/__cdc_meta__/owner/22318f4c626b339c 3398e8ae-814f-4a43-9659-7de7839827c5 /tidb/cdc/default/default/changefeed/info/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"upstream-id":7365754973417589185,"namespace":"default","changefeed-id":"9cbe3e2c-4ad8-4840-8911-e0dbef1c4359","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:31:32.691518234+08:00","start-ts":449570008778670081,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570011255930893} /tidb/cdc/default/default/changefeed/status/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":449570009434030104,"min-table-barrier-ts":449570012304506889,"admin-job-type":0} /tidb/cdc/default/default/task/position/3398e8ae-814f-4a43-9659-7de7839827c5/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754973417589185 {"id":7365754973417589185,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {UpstreamID:7365754973417589185 Namespace:default ID:9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:31:32.691518234 +0800 CST StartTs:449570008778670081 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031601b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570011255930893} {CheckpointTs:449570009434030104 MinTableBarrierTs:449570012304506889 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3398e8ae-814f-4a43-9659-7de7839827c5 {"id":"3398e8ae-814f-4a43-9659-7de7839827c5","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973494} /tidb/cdc/default/__cdc_meta__/capture/7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811 {"id":"7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973489} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c626b3344 7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811 /tidb/cdc/default/__cdc_meta__/owner/22318f4c626b339c 3398e8ae-814f-4a43-9659-7de7839827c5 /tidb/cdc/default/default/changefeed/info/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"upstream-id":7365754973417589185,"namespace":"default","changefeed-id":"9cbe3e2c-4ad8-4840-8911-e0dbef1c4359","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:31:32.691518234+08:00","start-ts":449570008778670081,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570011255930893} /tidb/cdc/default/default/changefeed/status/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":449570009434030104,"min-table-barrier-ts":449570012304506889,"admin-job-type":0} /tidb/cdc/default/default/task/position/3398e8ae-814f-4a43-9659-7de7839827c5/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754973417589185 {"id":7365754973417589185,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Mon May 6 13:31:37 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2537825380.out server --log-file /tmp/tidb_cdc_test/move_table/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data3 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/server_config_compatibility Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 22-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:31:37.636 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark6000] [2024/05/06 13:31:37.636 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark6000] table test.`simple-dash` not exists for 2-th check, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5497.out cli changefeed pause -c test PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' run task failed 2-th time, retry later [2024/05/06 13:31:38.737 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... *************************** 1. row *************************** count(distinct region_id): 2 Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 8-th check, retry later check diff failed 23-th time, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5530.out cli changefeed update -c test '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=true' --no-confirm VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318b4f7c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-63kf5-dnt52-vw515, pid:4109, start at 2024-05-06 13:31:36.33006562 +0800 CST m=+5.354328138 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:36.338 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:36.337 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:36.337 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318b50300005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-63kf5-dnt52-vw515, pid:4176, start at 2024-05-06 13:31:36.336188517 +0800 CST m=+5.290597501 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:36.343 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:36.332 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:36.332 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318b6f6c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-l2sg4-3h871-3slfr, pid:4211, start at 2024-05-06 13:31:38.379266088 +0800 CST m=+5.171173876 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:38.386 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:38.381 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:38.381 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318b6f6c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-l2sg4-3h871-3slfr, pid:4211, start at 2024-05-06 13:31:38.379266088 +0800 CST m=+5.171173876 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:38.386 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:38.381 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:38.381 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318b71b00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-l2sg4-3h871-3slfr, pid:4295, start at 2024-05-06 13:31:38.50473852 +0800 CST m=+5.247921079 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:38.513 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:38.476 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:38.476 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 13:31:39.636 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365754993661904829, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449570012963799044, "checkpoint_time": "2024-05-06 13:31:39.160", "error": null }' + echo '{ "upstream_id": 7365754993661904829, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449570012963799044, "checkpoint_time": "2024-05-06 13:31:39.160", "error": null }' { "upstream_id": 7365754993661904829, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449570012963799044, "checkpoint_time": "2024-05-06 13:31:39.160", "error": null } ++ echo '{' '"upstream_id":' 7365754993661904829, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449570012963799044, '"checkpoint_time":' '"2024-05-06' '13:31:39.160",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365754993661904829, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449570012963799044, '"checkpoint_time":' '"2024-05-06' '13:31:39.160",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 table partition_table.finish_mark exists check diff successfully Diff of changefeed config: {Type:update Path:[SinkURI] From:mysql://root@127.0.0.1:3306/?batch-dml-enable=false To:mysql://root@127.0.0.1:3306/?batch-dml-enable=true} changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 wait process cdc.test exit for 1-th time... Update changefeed config successfully! ID: test Info: {"upstream_id":7365754943604714677,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=true","create_time":"2024-05-06T13:31:22.690317203+08:00","start_ts":449570008613257220,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":0,"checkpoint_ts":449570009609404442,"checkpoint_time":"2024-05-06 13:31:26.364"} PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:40 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {UpstreamID:7365754973417589185 Namespace:default ID:9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:31:32.691518234 +0800 CST StartTs:449570008778670081 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003da53b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570011255930893} {CheckpointTs:449570009814139096 MinTableBarrierTs:449570013110075394 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3398e8ae-814f-4a43-9659-7de7839827c5 {"id":"3398e8ae-814f-4a43-9659-7de7839827c5","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973494} /tidb/cdc/default/__cdc_meta__/capture/7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811 {"id":"7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973489} /tidb/cdc/default/__cdc_meta__/capture/e6d591a2-27ae-435c-8275-1befd2fa5869 {"id":"e6d591a2-27ae-435c-8275-1befd2fa5869","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973497} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c626b3344 7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811 /tidb/cdc/default/__cdc_meta__/owner/22318f4c626b339c 3398e8ae-814f-4a43-9659-7de7839827c5 /tidb/cdc/default/__cdc_meta__/owner/22318f4c626b33c3 e6d591a2-27ae-435c-8275-1befd2fa5869 /tidb/cdc/default/default/changefeed/info/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"upstream-id":7365754973417589185,"namespace":"default","changefeed-id":"9cbe3e2c-4ad8-4840-8911-e0dbef1c4359","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:31:32.691518234+08:00","start-ts":449570008778670081,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570011255930893} /tidb/cdc/default/default/changefeed/status/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":449570009814139096,"min-table-barrier-ts":449570013110075394,"admin-job-type":0} /tidb/cdc/default/default/task/position/3398e8ae-814f-4a43-9659-7de7839827c5/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e6d591a2-27ae-435c-8275-1befd2fa5869/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754973417589185 {"id":7365754973417589185,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {UpstreamID:7365754973417589185 Namespace:default ID:9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:31:32.691518234 +0800 CST StartTs:449570008778670081 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003da53b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570011255930893} {CheckpointTs:449570009814139096 MinTableBarrierTs:449570013110075394 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3398e8ae-814f-4a43-9659-7de7839827c5 {"id":"3398e8ae-814f-4a43-9659-7de7839827c5","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973494} /tidb/cdc/default/__cdc_meta__/capture/7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811 {"id":"7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973489} /tidb/cdc/default/__cdc_meta__/capture/e6d591a2-27ae-435c-8275-1befd2fa5869 {"id":"e6d591a2-27ae-435c-8275-1befd2fa5869","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973497} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c626b3344 7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811 /tidb/cdc/default/__cdc_meta__/owner/22318f4c626b339c 3398e8ae-814f-4a43-9659-7de7839827c5 /tidb/cdc/default/__cdc_meta__/owner/22318f4c626b33c3 e6d591a2-27ae-435c-8275-1befd2fa5869 /tidb/cdc/default/default/changefeed/info/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"upstream-id":7365754973417589185,"namespace":"default","changefeed-id":"9cbe3e2c-4ad8-4840-8911-e0dbef1c4359","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:31:32.691518234+08:00","start-ts":449570008778670081,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570011255930893} /tidb/cdc/default/default/changefeed/status/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":449570009814139096,"min-table-barrier-ts":449570013110075394,"admin-job-type":0} /tidb/cdc/default/default/task/position/3398e8ae-814f-4a43-9659-7de7839827c5/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e6d591a2-27ae-435c-8275-1befd2fa5869/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754973417589185 {"id":7365754973417589185,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {UpstreamID:7365754973417589185 Namespace:default ID:9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:31:32.691518234 +0800 CST StartTs:449570008778670081 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003da53b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570011255930893} {CheckpointTs:449570009814139096 MinTableBarrierTs:449570013110075394 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3398e8ae-814f-4a43-9659-7de7839827c5 {"id":"3398e8ae-814f-4a43-9659-7de7839827c5","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973494} /tidb/cdc/default/__cdc_meta__/capture/7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811 {"id":"7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973489} /tidb/cdc/default/__cdc_meta__/capture/e6d591a2-27ae-435c-8275-1befd2fa5869 {"id":"e6d591a2-27ae-435c-8275-1befd2fa5869","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973497} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c626b3344 7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811 /tidb/cdc/default/__cdc_meta__/owner/22318f4c626b339c 3398e8ae-814f-4a43-9659-7de7839827c5 /tidb/cdc/default/__cdc_meta__/owner/22318f4c626b33c3 e6d591a2-27ae-435c-8275-1befd2fa5869 /tidb/cdc/default/default/changefeed/info/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"upstream-id":7365754973417589185,"namespace":"default","changefeed-id":"9cbe3e2c-4ad8-4840-8911-e0dbef1c4359","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:31:32.691518234+08:00","start-ts":449570008778670081,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570011255930893} /tidb/cdc/default/default/changefeed/status/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":449570009814139096,"min-table-barrier-ts":449570013110075394,"admin-job-type":0} /tidb/cdc/default/default/task/position/3398e8ae-814f-4a43-9659-7de7839827c5/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e6d591a2-27ae-435c-8275-1befd2fa5869/9cbe3e2c-4ad8-4840-8911-e0dbef1c4359 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754973417589185 {"id":7365754973417589185,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x wait process cdc.test exit for 2-th time... [2024/05/06 13:31:40.733 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark7000] [2024/05/06 13:31:40.733 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark7000] [2024/05/06 13:31:40.750 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] [2024/05/06 13:31:40.838 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark5000] [ts=449570012891709445] [2024/05/06 13:31:40.838 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark5000] [2024/05/06 13:31:40.838 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570012891709445] [2024/05/06 13:31:40.840 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:40.843 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:31:40.847 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:40.924 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table move_table.usertable exists go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 table mark.finish_mark_3 not exists for 9-th check, retry later go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading github.com/gin-gonic/gin v1.9.1 wait process cdc.test exit for 3-th time... Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12722.out cli changefeed --changefeed-id custom-changefeed-name pause go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:31:41 CST 2024] <<<<<< run test case partition_table success! >>>>>> + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5571.out cli changefeed resume -c test Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/aws/smithy-go v1.13.5 [2024/05/06 13:31:41.735 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff successfully [Mon May 6 13:31:41 CST 2024] <<<<<< START cdc server in processor_etcd_worker_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/orchestrator/ProcessorEtcdDelay=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_etcd_worker_delay.55425544.out server --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 PASS wait process cdc.test exit for 1-th time... go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release coverage: 2.1% of statements in github.com/pingcap/tiflow/... go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' wait process cdc.test exit for 2-th time... [2024/05/06 13:31:42.733 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 [Mon May 6 13:31:42 CST 2024] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.54495451.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 13:31:43.139 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark8000] [2024/05/06 13:31:43.139 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark8000] Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 10-th check, retry later split_and_random_merge scale: 80 + set +x wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:31:43 CST 2024] <<<<<< run test case clustered_index success! >>>>>> + set +x "is-owner": true, run task successfully nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/06 13:31:43.821 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] [2024/05/06 13:31:43.924 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark6000] [ts=449570013808951301] [2024/05/06 13:31:43.925 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark6000] [2024/05/06 13:31:43.925 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570013808951301] [2024/05/06 13:31:43.928 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:43.932 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:31:43.937 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:44.024 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table batch_update_to_no_batch.v1 exists table batch_update_to_no_batch.recover_and_insert not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:44 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbe72426-cd2f-413f-9d3b-6f90506e5501 {"id":"dbe72426-cd2f-413f-9d3b-6f90506e5501","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973501} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c62b3caca dbe72426-cd2f-413f-9d3b-6f90506e5501 /tidb/cdc/default/default/upstream/7365755050495697855 {"id":7365755050495697855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbe72426-cd2f-413f-9d3b-6f90506e5501 {"id":"dbe72426-cd2f-413f-9d3b-6f90506e5501","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973501} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c62b3caca dbe72426-cd2f-413f-9d3b-6f90506e5501 /tidb/cdc/default/default/upstream/7365755050495697855 {"id":7365755050495697855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbe72426-cd2f-413f-9d3b-6f90506e5501 {"id":"dbe72426-cd2f-413f-9d3b-6f90506e5501","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973501} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c62b3caca dbe72426-cd2f-413f-9d3b-6f90506e5501 /tidb/cdc/default/default/upstream/7365755050495697855 {"id":7365755050495697855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/06 13:31:44.831 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] table mark.finish_mark_3 not exists for 11-th check, retry later [2024/05/06 13:31:45.139 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:45 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6283651d-9c57-45dd-9187-649bae05617a {"id":"6283651d-9c57-45dd-9187-649bae05617a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973502} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c62a5f419 6283651d-9c57-45dd-9187-649bae05617a /tidb/cdc/default/default/upstream/7365755039354231441 {"id":7365755039354231441,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6283651d-9c57-45dd-9187-649bae05617a {"id":"6283651d-9c57-45dd-9187-649bae05617a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973502} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c62a5f419 6283651d-9c57-45dd-9187-649bae05617a /tidb/cdc/default/default/upstream/7365755039354231441 {"id":7365755039354231441,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6283651d-9c57-45dd-9187-649bae05617a {"id":"6283651d-9c57-45dd-9187-649bae05617a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973502} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c62a5f419 6283651d-9c57-45dd-9187-649bae05617a /tidb/cdc/default/default/upstream/7365755039354231441 {"id":7365755039354231441,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.5501.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ [2024/05/06 13:31:39.725 +08:00] [INFO] [main.go:86] ["running ddl test: 0 createDropSchemaDDL"] [2024/05/06 13:31:40.253 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 13:31:40.260 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 13:31:40.741 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 13:31:40.743 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 13:31:40.743 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 13:31:40.744 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 13:31:41.218 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/06 13:31:41.220 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 13:31:41.688 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 13:31:41.690 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 13:31:41.695 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 13:31:41.701 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 13:31:42.170 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 13:31:42.176 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/06 13:31:42.681 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 13:31:42.684 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 13:31:43.066 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 13:31:43.072 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/06 13:31:43.160 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 13:31:43.549 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 13:31:43.645 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 13:31:43.650 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 13:31:44.038 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 13:31:44.045 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 13:31:44.129 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/06 13:31:44.533 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 13:31:44.603 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 13:31:44.606 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [mysql] 2024/05/06 13:31:44 connection.go:299: invalid connection ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully Create changefeed successfully! ID: a10b0548-93be-48f1-add2-55751cddcdad Info: {"upstream_id":7365755039354231441,"namespace":"default","id":"a10b0548-93be-48f1-add2-55751cddcdad","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:31:45.837046098+08:00","start_ts":449570014687920131,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570014687920131,"checkpoint_ts":449570014687920131,"checkpoint_time":"2024-05-06 13:31:45.737"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table batch_update_to_no_batch.recover_and_insert not exists for 2-th check, retry later [2024/05/06 13:31:46.234 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark9000] [2024/05/06 13:31:46.234 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark9000] [2024/05/06 13:31:46.847 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] table mark.finish_mark_3 not exists for 12-th check, retry later go: downloading github.com/ardielle/ardielle-go v1.5.2 [2024/05/06 13:31:46.937 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark7000] [ts=449570014674288645] [2024/05/06 13:31:46.937 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark7000] [2024/05/06 13:31:46.937 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570014674288645] [2024/05/06 13:31:47.023 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:47.027 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:31:47.032 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:47.036 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] + set +x [2024/05/06 13:31:47.226 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table batch_add_table.finish_mark not exists for 1-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/06 13:31:48.234 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] ./t/bigint.test: ok! 138 test cases passed, take time 14.920841421 s time="2024-05-06T13:31:46+08:00" level=info msg="run test [bigint] ok" time="2024-05-06T13:31:46+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0000e89b0 {{} {} 0} {0 0} [0xc0002f5170] map[] 0 1 0xc000500060 false map[0xc0002f5170:map[0xc0002f5170:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" ./t/composite_index.test: ok! 12 test cases passed, take time 1.553992904 s time="2024-05-06T13:31:48+08:00" level=info msg="run test [composite_index] ok" time="2024-05-06T13:31:48+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc000298018 {{} {} 0} {0 0} [0xc0002f4000] map[] 0 1 0xc00041c000 false map[0xc0002f4000:map[0xc0002f4000:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365754993661904829, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449570013488087044, "checkpoint_time": "2024-05-06 13:31:41.160", "error": null }' + echo '{ "upstream_id": 7365754993661904829, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449570013488087044, "checkpoint_time": "2024-05-06 13:31:41.160", "error": null }' { "upstream_id": 7365754993661904829, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449570013488087044, "checkpoint_time": "2024-05-06 13:31:41.160", "error": null } ++ echo '{' '"upstream_id":' 7365754993661904829, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449570013488087044, '"checkpoint_time":' '"2024-05-06' '13:31:41.160",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365754993661904829, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449570013488087044, '"checkpoint_time":' '"2024-05-06' '13:31:41.160",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12813.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table batch_update_to_no_batch.recover_and_insert exists table batch_update_to_no_batch.finish_mark not exists for 1-th check, retry later Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7365754993661904829,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:31:33.949926786+08:00","start_ts":449570010237239297,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":0,"checkpoint_ts":449570013488087044,"checkpoint_time":"2024-05-06 13:31:41.160"} PASS coverage: 2.8% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 13-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318c0b640015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d, pid:10516, start at 2024-05-06 13:31:48.36109244 +0800 CST m=+5.263709119 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:48.372 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:48.363 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:48.363 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table batch_add_table.finish_mark not exists for 2-th check, retry later [2024/05/06 13:31:49.240 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] [2024/05/06 13:31:49.326 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark8000] [ts=449570015522586629] [2024/05/06 13:31:49.326 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark8000] [2024/05/06 13:31:49.326 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570015522586629] [2024/05/06 13:31:49.336 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:49.340 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:31:49.425 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:49.430 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:31:49.535 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark10000] [2024/05/06 13:31:49.535 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark10000] *************************** 1. row *************************** count(distinct region_id): 9 ./t/concurrent_ddl.test: ok! 8 test cases passed, take time 1.028442741 s time="2024-05-06T13:31:49+08:00" level=info msg="run test [concurrent_ddl] ok" time="2024-05-06T13:31:49+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc000446060 {{} {} 0} {0 0} [0xc000452090] map[] 0 1 0xc000500360 false map[0xc000452090:map[0xc000452090:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12851.out cli changefeed --changefeed-id custom-changefeed-name resume [2024/05/06 13:31:50.321 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] empty select id, val from test.availability1 where id=1 run task successfully test_owner_cleanup_stale_tasks pass wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:31:49.806 +08:00] [INFO] [main.go:86] ["running ddl test: 1 truncateDDL"] [2024/05/06 13:31:50.096 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 13:31:50.106 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 13:31:50.290 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 13:31:50.291 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 13:31:50.292 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 13:31:50.293 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 13:31:50.479 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 13:31:50.479 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/06 13:31:50.656 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 13:31:50.663 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 13:31:50.671 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 13:31:50.673 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] table batch_update_to_no_batch.finish_mark not exists for 2-th check, retry later [2024/05/06 13:31:50.834 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 13:31:50.858 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... [2024/05/06 13:31:51.023 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 13:31:51.033 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 13:31:51.051 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 13:31:51.060 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318c0b640015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d, pid:10516, start at 2024-05-06 13:31:48.36109244 +0800 CST m=+5.263709119 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:48.372 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:48.363 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:48.363 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318c0cdc000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-k2zxd-03tw6-sn75d, pid:10606, start at 2024-05-06 13:31:48.419254744 +0800 CST m=+5.246412860 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:48.428 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:48.407 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:48.407 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/error.log [2024/05/06 13:31:51.206 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 13:31:51.244 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 13:31:51.398 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 13:31:51.408 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] table mark.finish_mark_3 not exists for 14-th check, retry later arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 13:31:51.528 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 13:31:51.548 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 13:31:51.595 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_owner_retryable_error [Mon May 6 13:31:51 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/capture/capture-campaign-compacted-error=1*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.41064108.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 13:31:51.535 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/06 13:31:51.797 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 13:31:51.855 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 13:31:51.873 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] table batch_add_table.finish_mark exists check diff successfully [2024/05/06 13:31:51.993 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 13:31:52.017 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/06 13:31:52.051 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/06 13:31:52.189 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] wait process cdc.test exit for 1-th time... [2024/05/06 13:31:52.238 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 13:31:52.335 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/06 13:31:52.377 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 13:31:52.407 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/06 13:31:52.335 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] [2024/05/06 13:31:52.351 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark9000] [ts=449570016361447427] [2024/05/06 13:31:52.351 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark9000] [2024/05/06 13:31:52.351 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570016361447427] [2024/05/06 13:31:52.353 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:52.356 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:31:52.360 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:52.365 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318c32180013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5155r-hgjhp-wsj0l, pid:23516, start at 2024-05-06 13:31:50.829885308 +0800 CST m=+6.654452029 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:50.839 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:50.840 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:50.840 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318c32180013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5155r-hgjhp-wsj0l, pid:23516, start at 2024-05-06 13:31:50.829885308 +0800 CST m=+6.654452029 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:50.839 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:50.840 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:50.840 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318c1dbc0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5155r-hgjhp-wsj0l, pid:23585, start at 2024-05-06 13:31:49.523237864 +0800 CST m=+5.289808005 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:49.531 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:49.537 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:49.537 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x [2024/05/06 13:31:52.513 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/06 13:31:52.568 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/06 13:31:52.707 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 13:31:52.727 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/06 13:31:52.533 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark11000] [2024/05/06 13:31:52.534 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark11000] wait process cdc.test exit for 2-th time... [2024/05/06 13:31:52.758 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 13:31:52.790 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/06 13:31:52.904 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/06 13:31:52.956 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/06 13:31:53.087 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/06 13:31:53.106 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/06 13:31:53.144 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/06 13:31:53.190 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:31:53 CST 2024] <<<<<< run test case batch_add_table success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_nfs Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 15-th check, retry later [2024/05/06 13:31:53.273 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/06 13:31:53.334 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/06 13:31:53.459 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/06 13:31:53.486 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.cli.11876.out cli tso query --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 13:31:53.621 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/06 13:31:53.516 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/06 13:31:53.567 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/06 13:31:53.652 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/06 13:31:53.707 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m table batch_update_to_no_batch.finish_mark not exists for 3-th check, retry later [2024/05/06 13:31:53.832 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/06 13:31:53.865 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/06 13:31:53.896 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/06 13:31:53.959 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/06 13:31:54.012 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/06 13:31:54.094 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/06 13:31:54.205 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/06 13:31:54.244 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/06 13:31:54.271 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/06 13:31:54.334 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/06 13:31:54.396 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/06 13:31:54.446 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/06 13:31:54.534 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] [Pipeline] } [2024/05/06 13:31:54.583 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/06 13:31:54.623 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/06 13:31:54.627 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/06 13:31:54.689 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc already exists) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:54 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e99320 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570015625084939 MinTableBarrierTs:449570016935804939 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570016935804939, checkpointTs: 449570016935804939, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570016935804936, checkpointTs: 449570016935804936, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570016935804939, checkpointTs: 449570016935804939, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b5b09aa4-b43f-4d0a-b16b-857609eff8db {"id":"b5b09aa4-b43f-4d0a-b16b-857609eff8db","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973511} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4181 b5b09aa4-b43f-4d0a-b16b-857609eff8db /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570016935804936,"min-table-barrier-ts":449570016935804939,"admin-job-type":0} /tidb/cdc/default/default/task/position/b5b09aa4-b43f-4d0a-b16b-857609eff8db/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e99320 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570015625084939 MinTableBarrierTs:449570016935804939 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570016935804939, checkpointTs: 449570016935804939, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570016935804936, checkpointTs: 449570016935804936, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570016935804939, checkpointTs: 449570016935804939, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b5b09aa4-b43f-4d0a-b16b-857609eff8db {"id":"b5b09aa4-b43f-4d0a-b16b-857609eff8db","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973511} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4181 b5b09aa4-b43f-4d0a-b16b-857609eff8db /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570016935804936,"min-table-barrier-ts":449570016935804939,"admin-job-type":0} /tidb/cdc/default/default/task/position/b5b09aa4-b43f-4d0a-b16b-857609eff8db/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e99320 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570015625084939 MinTableBarrierTs:449570016935804939 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570016935804939, checkpointTs: 449570016935804939, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570016935804936, checkpointTs: 449570016935804936, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570016935804939, checkpointTs: 449570016935804939, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b5b09aa4-b43f-4d0a-b16b-857609eff8db {"id":"b5b09aa4-b43f-4d0a-b16b-857609eff8db","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973511} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4181 b5b09aa4-b43f-4d0a-b16b-857609eff8db /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570016935804936,"min-table-barrier-ts":449570016935804939,"admin-job-type":0} /tidb/cdc/default/default/task/position/b5b09aa4-b43f-4d0a-b16b-857609eff8db/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.24956.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 13:31:54.771 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] table region_merge.t1 exists check diff failed 1-th time, retry later [Pipeline] // cache [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table mark.finish_mark_3 not exists for 16-th check, retry later [Pipeline] // dir + set +x + tso='449570016791887873 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570016791887873 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:31:55 CST 2024] <<<<<< START cdc server in server_config_compatibility case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.1192111923.out server --log-file /tmp/tidb_cdc_test/server_config_compatibility/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/server_config_compatibility/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/server_config_compatibility/conf/server.toml ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // stage [Pipeline] } + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365754993661904829, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449570017157840902, "checkpoint_time": "2024-05-06 13:31:55.159", "error": null }' + echo '{ "upstream_id": 7365754993661904829, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449570017157840902, "checkpoint_time": "2024-05-06 13:31:55.159", "error": null }' { "upstream_id": 7365754993661904829, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449570017157840902, "checkpoint_time": "2024-05-06 13:31:55.159", "error": null } ++ echo '{' '"upstream_id":' 7365754993661904829, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449570017157840902, '"checkpoint_time":' '"2024-05-06' '13:31:55.159",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365754993661904829, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449570017157840902, '"checkpoint_time":' '"2024-05-06' '13:31:55.159",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12940.out cli changefeed --changefeed-id custom-changefeed-name remove table batch_update_to_no_batch.finish_mark not exists for 4-th check, retry later [2024/05/06 13:31:55.635 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark12000] [2024/05/06 13:31:55.635 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark12000] [2024/05/06 13:31:55.635 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/06 13:31:55.735 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark10000] [ts=449570017200308229] [2024/05/06 13:31:55.735 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark10000] [2024/05/06 13:31:55.735 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570017200308229] [2024/05/06 13:31:55.737 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:55.825 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:31:55.829 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:55.834 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } "is-owner": true, run task successfully + set +x + tso='449570017074216961 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570017074216961 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:31:56 CST 2024] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.2499424996.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 449570017157840902 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/ PASS check diff successfully [Pipeline] // podTemplate ./t/date_formats.test: ok! 99 test cases passed, take time 5.315228013 s time="2024-05-06T13:31:55+08:00" level=info msg="run test [date_formats] ok" time="2024-05-06T13:31:55+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0002aa440 {{} {} 0} {0 0} [0xc000405d40] map[] 0 1 0xc000282f00 false map[0xc000405d40:map[0xc000405d40:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" [Pipeline] } coverage: 2.0% of statements in github.com/pingcap/tiflow/... [Pipeline] // withEnv [Pipeline] } [2024/05/06 13:31:56.621 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] // stage [2024/05/06 13:31:55.720 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc001a421e0/127.0.0.1:2479] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"] [2024/05/06 13:31:55.744 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc001c045a0/127.0.0.1:2379] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"] [2024/05/06 13:31:55.908 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc000e7a1e0/127.0.0.1:2379] [attempt=0] [error="rpc error: code = NotFound desc = etcdserver: requested lease not found"] [2024/05/06 13:31:55.912 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc0015d61e0/127.0.0.1:2479] [attempt=0] [error="rpc error: code = NotFound desc = etcdserver: requested lease not found"] wait process cdc.test exit for 1-th time... [Pipeline] } Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 17-th check, retry later wait process cdc.test exit for 2-th time... owner pid: 4111 owner id b5b09aa4-b43f-4d0a-b16b-857609eff8db [Mon May 6 13:31:57 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/owner-run-with-error=1*return(true);github.com/pingcap/tiflow/cdc/capture/capture-resign-failed=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.42454247.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table batch_update_to_no_batch.finish_mark exists check diff successfully [2024/05/06 13:31:57.635 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] + set +x wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:31:58 CST 2024] <<<<<< run test case region_merge success! >>>>>> [2024/05/06 13:31:58.636 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] [2024/05/06 13:31:58.651 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark11000] [ts=449570018003255301] [2024/05/06 13:31:58.651 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark11000] [2024/05/06 13:31:58.651 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570018003255301] [2024/05/06 13:31:58.653 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:58.656 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:31:58.659 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:31:58.664 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Verifying downstream PD is started... wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:58 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27cf48fd-e6e5-488c-be4f-07cb323d98d4 {"id":"27cf48fd-e6e5-488c-be4f-07cb323d98d4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973515} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c62d8a4ef 27cf48fd-e6e5-488c-be4f-07cb323d98d4 /tidb/cdc/default/default/upstream/7365755098866740122 {"id":7365755098866740122,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27cf48fd-e6e5-488c-be4f-07cb323d98d4 {"id":"27cf48fd-e6e5-488c-be4f-07cb323d98d4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973515} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c62d8a4ef 27cf48fd-e6e5-488c-be4f-07cb323d98d4 /tidb/cdc/default/default/upstream/7365755098866740122 {"id":7365755098866740122,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27cf48fd-e6e5-488c-be4f-07cb323d98d4 {"id":"27cf48fd-e6e5-488c-be4f-07cb323d98d4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973515} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c62d8a4ef 27cf48fd-e6e5-488c-be4f-07cb323d98d4 /tidb/cdc/default/default/upstream/7365755098866740122 {"id":7365755098866740122,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.cli.11982.out cli changefeed create --start-ts=449570016791887873 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ [2024/05/06 13:31:58.934 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark13000] [2024/05/06 13:31:58.934 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark13000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 18-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:31:59 CST 2024] <<<<<< run test case batch_update_to_no_batch success! >>>>>> Create changefeed successfully! ID: 54b45010-272e-4a39-850a-60cb9ae1ea62 Info: {"upstream_id":7365755098866740122,"namespace":"default","id":"54b45010-272e-4a39-850a-60cb9ae1ea62","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:31:58.939903359+08:00","start_ts":449570016791887873,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570016791887873,"checkpoint_ts":449570016791887873,"checkpoint_time":"2024-05-06 13:31:53.763"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:31:59 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ea17b88c-7cdf-420d-96b9-496ccb3d1474 {"id":"ea17b88c-7cdf-420d-96b9-496ccb3d1474","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973516} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c62decbd3 ea17b88c-7cdf-420d-96b9-496ccb3d1474 /tidb/cdc/default/default/upstream/7365755100392136818 {"id":7365755100392136818,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ea17b88c-7cdf-420d-96b9-496ccb3d1474 {"id":"ea17b88c-7cdf-420d-96b9-496ccb3d1474","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973516} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c62decbd3 ea17b88c-7cdf-420d-96b9-496ccb3d1474 /tidb/cdc/default/default/upstream/7365755100392136818 {"id":7365755100392136818,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ea17b88c-7cdf-420d-96b9-496ccb3d1474 {"id":"ea17b88c-7cdf-420d-96b9-496ccb3d1474","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973516} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c62decbd3 ea17b88c-7cdf-420d-96b9-496ccb3d1474 /tidb/cdc/default/default/upstream/7365755100392136818 {"id":7365755100392136818,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.25052.out cli changefeed create --start-ts=449570017074216961 --sink-uri=mysql://root@127.0.0.1:3306/ [2024/05/06 13:31:59.722 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] Create changefeed successfully! ID: b1f174aa-01a8-465a-bf3e-3ce88e6edb4e Info: {"upstream_id":7365755100392136818,"namespace":"default","id":"b1f174aa-01a8-465a-bf3e-3ce88e6edb4e","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-06T13:31:59.8823367+08:00","start_ts":449570017074216961,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570017074216961,"checkpoint_ts":449570017074216961,"checkpoint_time":"2024-05-06 13:31:54.840"} PASS VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318caf400019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv, pid:10377, start at 2024-05-06 13:31:58.838277531 +0800 CST m=+5.203787276 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:58.845 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:58.849 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:58.849 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/06 13:31:59.902 +08:00] [INFO] [main.go:86] ["running ddl test: 2 addDropColumnDDL"] [2024/05/06 13:32:00.124 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 13:32:00.127 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 13:32:00.312 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 13:32:00.314 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 13:32:00.321 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 13:32:00.323 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 13:32:00.502 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 13:32:00.504 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] + set +x Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 13:32:00.690 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 13:32:00.692 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 13:32:00.700 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 13:32:00.702 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later table mark.finish_mark_3 not exists for 19-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:32:00 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002738000 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570018246262800 MinTableBarrierTs:449570018508668943 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4541caa1-76b9-4315-8286-9ff20cfdabaf {"id":"4541caa1-76b9-4315-8286-9ff20cfdabaf","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973517} /tidb/cdc/default/__cdc_meta__/capture/b5b09aa4-b43f-4d0a-b16b-857609eff8db {"id":"b5b09aa4-b43f-4d0a-b16b-857609eff8db","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973511} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4181 b5b09aa4-b43f-4d0a-b16b-857609eff8db /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac41d2 4541caa1-76b9-4315-8286-9ff20cfdabaf /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570018246262800,"min-table-barrier-ts":449570018508668943,"admin-job-type":0} /tidb/cdc/default/default/task/position/4541caa1-76b9-4315-8286-9ff20cfdabaf/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b5b09aa4-b43f-4d0a-b16b-857609eff8db/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002738000 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570018246262800 MinTableBarrierTs:449570018508668943 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4541caa1-76b9-4315-8286-9ff20cfdabaf {"id":"4541caa1-76b9-4315-8286-9ff20cfdabaf","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973517} /tidb/cdc/default/__cdc_meta__/capture/b5b09aa4-b43f-4d0a-b16b-857609eff8db {"id":"b5b09aa4-b43f-4d0a-b16b-857609eff8db","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973511} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4181 b5b09aa4-b43f-4d0a-b16b-857609eff8db /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac41d2 4541caa1-76b9-4315-8286-9ff20cfdabaf /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570018246262800,"min-table-barrier-ts":449570018508668943,"admin-job-type":0} /tidb/cdc/default/default/task/position/4541caa1-76b9-4315-8286-9ff20cfdabaf/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b5b09aa4-b43f-4d0a-b16b-857609eff8db/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002738000 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570018246262800 MinTableBarrierTs:449570018508668943 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4541caa1-76b9-4315-8286-9ff20cfdabaf {"id":"4541caa1-76b9-4315-8286-9ff20cfdabaf","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973517} /tidb/cdc/default/__cdc_meta__/capture/b5b09aa4-b43f-4d0a-b16b-857609eff8db {"id":"b5b09aa4-b43f-4d0a-b16b-857609eff8db","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973511} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4181 b5b09aa4-b43f-4d0a-b16b-857609eff8db /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac41d2 4541caa1-76b9-4315-8286-9ff20cfdabaf /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570018246262800,"min-table-barrier-ts":449570018508668943,"admin-job-type":0} /tidb/cdc/default/default/task/position/4541caa1-76b9-4315-8286-9ff20cfdabaf/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b5b09aa4-b43f-4d0a-b16b-857609eff8db/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "b5b09aa4-b43f-4d0a-b16b-857609eff8db" | grep id [2024/05/06 13:32:00.899 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/06 13:32:00.918 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 13:32:00.934 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] [2024/05/06 13:32:01.086 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 13:32:01.088 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/06 13:32:01.118 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 13:32:01.120 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 13:32:01.275 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13020.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name + set +x [2024/05/06 13:32:01.320 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 13:32:01.470 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 13:32:01.472 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 13:32:01.523 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 13:32:01.525 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365754993661904829,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:32:01.448404995+08:00","start_ts":449570018770288646,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570018770288646,"checkpoint_ts":449570018770288646,"checkpoint_time":"2024-05-06 13:32:01.310"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:32:01.656 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 13:32:01.709 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/06 13:32:01.738 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318caf400019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv, pid:10377, start at 2024-05-06 13:31:58.838277531 +0800 CST m=+5.203787276 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:58.845 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:58.849 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:58.849 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318caff00008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-60ww2-k02p4-rnhlv, pid:10461, start at 2024-05-06 13:31:58.851759114 +0800 CST m=+5.169516912 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:33:58.859 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:31:58.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:21:58.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 13:32:01.859 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 13:32:01.861 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/06 13:32:01.888 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 13:32:01.890 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/06 13:32:01.949 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark14000] [2024/05/06 13:32:01.949 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark14000] [2024/05/06 13:32:02.046 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/06 13:32:02.064 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/06 13:32:02.235 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 13:32:02.237 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/06 13:32:02.248 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 13:32:02.250 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] "cluster-id": "default" "id": "4541caa1-76b9-4315-8286-9ff20cfdabaf", "cluster-id": "default" run task successfully [2024/05/06 13:32:02.415 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/06 13:32:02.433 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later [2024/05/06 13:32:02.602 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 13:32:02.604 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/06 13:32:02.613 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 13:32:02.615 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/06 13:32:02.778 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/06 13:32:02.788 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/06 13:32:02.946 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] [2024/05/06 13:32:02.952 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/06 13:32:02.954 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/06 13:32:02.964 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/06 13:32:02.966 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:32:03.139 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/06 13:32:03.143 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318ce5280017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz, pid:7085, start at 2024-05-06 13:32:02.287169061 +0800 CST m=+5.348446682 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:02.296 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:02.300 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:02.300 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 20-th check, retry later [2024/05/06 13:32:03.325 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/06 13:32:03.327 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/06 13:32:03.332 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/06 13:32:03.334 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/06 13:32:03.509 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/06 13:32:03.518 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] capture_id: 4541caa1-76b9-4315-8286-9ff20cfdabaf % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 35 100 35 0 0 27559 0 --:--:-- --:--:-- --:--:-- 35000 { "status": true, "message": "" }ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep 'b5b09aa4-b43f-4d0a-b16b-857609eff8db' b5b09aa4-b43f-4d0a-b16b-857609eff8db run task successfully ps -C cdc.test -o pid= | awk '{print $1}' | wc -l | grep 2 2 run task successfully test_owner_retryable_error pass [2024/05/06 13:32:03.697 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/06 13:32:03.699 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/06 13:32:03.707 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/06 13:32:03.709 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/06 13:32:03.821 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] [2024/05/06 13:32:03.841 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark12000] [ts=449570019192602629] [2024/05/06 13:32:03.842 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark12000] [2024/05/06 13:32:03.842 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570019192602629] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.11767.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 13:32:03.934 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/06 13:32:03.936 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/06 13:32:03.924 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:32:03.932 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:32:03.949 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] [2024/05/06 13:32:04.025 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:32:04.032 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] wait process cdc.test exit for 1-th time... [2024/05/06 13:32:04.120 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/06 13:32:04.121 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/06 13:32:04.122 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/06 13:32:04.123 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/06 13:32:04.296 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/06 13:32:04.297 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/06 13:32:04.333 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark15000] [2024/05/06 13:32:04.333 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark15000] [2024/05/06 13:32:04.469 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/06 13:32:04.471 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/05/06 13:32:04.473 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/06 13:32:04.475 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/06 13:32:04.643 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/05/06 13:32:04.644 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/05/06 13:32:04.816 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/05/06 13:32:04.818 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/05/06 13:32:04.828 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/05/06 13:32:04.830 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] wait process cdc.test exit for 2-th time... [2024/05/06 13:32:04.959 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check data successfully wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318ce5280017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz, pid:7085, start at 2024-05-06 13:32:02.287169061 +0800 CST m=+5.348446682 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:02.296 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:02.300 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:02.300 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318ce83c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz, pid:7163, start at 2024-05-06 13:32:02.447765552 +0800 CST m=+5.459744058 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:02.455 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:02.447 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:02.447 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_gap_between_watch_capture [Mon May 6 13:32:05 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.44254427.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 21-th check, retry later wait process cdc.test exit for 2-th time... + set +x + tso='449570019540992002 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570019540992002 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:32:05 CST 2024] <<<<<< START cdc server in resourcecontrol case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.1180311805.out server --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resourcecontrol/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365754993661904829, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449570019779543045, "checkpoint_time": "2024-05-06 13:32:05.160", "error": null }' + echo '{ "upstream_id": 7365754993661904829, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449570019779543045, "checkpoint_time": "2024-05-06 13:32:05.160", "error": null }' { "upstream_id": 7365754993661904829, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449570019779543045, "checkpoint_time": "2024-05-06 13:32:05.160", "error": null } ++ echo '{' '"upstream_id":' 7365754993661904829, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449570019779543045, '"checkpoint_time":' '"2024-05-06' '13:32:05.160",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365754993661904829, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449570019779543045, '"checkpoint_time":' '"2024-05-06' '13:32:05.160",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] [2024/05/06 13:32:06.038 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:32:06 CST 2024] <<<<<< run test case server_config_compatibility success! >>>>>> ./t/drop.test: ok! 48 test cases passed, take time 8.728605281 s time="2024-05-06T13:32:04+08:00" level=info msg="run test [drop] ok" time="2024-05-06T13:32:04+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0003261a0 {{} {} 0} {0 0} [0xc000405ef0] map[] 0 1 0xc00032a8a0 false map[0xc000405ef0:map[0xc000405ef0:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" ./t/mysql_replace.test: ok! 16 test cases passed, take time 1.047578533 s time="2024-05-06T13:32:06+08:00" level=info msg="run test [mysql_replace] ok" Great, All tests passed mysqltest end [2024/05/06 13:32:06.333 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13137.out cli unsafe delete-service-gc-safepoint ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 1-th check, retry later [2024/05/06 13:32:07.035 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 22-th check, retry later [2024/05/06 13:32:07.221 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark13000] [ts=449570020057677828] [2024/05/06 13:32:07.221 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark13000] [2024/05/06 13:32:07.222 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570020057677828] [2024/05/06 13:32:07.226 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:32:07.230 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:32:07.235 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:32:07.324 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:32:07.636 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark16000] [2024/05/06 13:32:07.636 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark16000] [Mon May 6 13:32:07 CST 2024] <<<<<< START cdc server in consistent_replicate_nfs case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_nfs.84648466.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_rocks/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 13:32:08.052 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] [2024/05/06 13:32:08.346 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13171.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/common_1/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) reset and all metadata truncated in PD! PASS table test.finish_mark not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:32:08 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/816fa5b1-36ee-4332-913e-53aebc672cf8 {"id":"816fa5b1-36ee-4332-913e-53aebc672cf8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973526} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c6303c1d6 816fa5b1-36ee-4332-913e-53aebc672cf8 /tidb/cdc/default/default/upstream/7365755140632559861 {"id":7365755140632559861,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/816fa5b1-36ee-4332-913e-53aebc672cf8 {"id":"816fa5b1-36ee-4332-913e-53aebc672cf8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973526} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c6303c1d6 816fa5b1-36ee-4332-913e-53aebc672cf8 /tidb/cdc/default/default/upstream/7365755140632559861 {"id":7365755140632559861,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/816fa5b1-36ee-4332-913e-53aebc672cf8 {"id":"816fa5b1-36ee-4332-913e-53aebc672cf8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973526} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c6303c1d6 816fa5b1-36ee-4332-913e-53aebc672cf8 /tidb/cdc/default/default/upstream/7365755140632559861 {"id":7365755140632559861,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.11859.out cli changefeed create --start-ts=449570019540992002 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ coverage: 2.0% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 23-th check, retry later Create changefeed successfully! ID: 818e736f-628e-43d2-a8ae-055cb251e392 Info: {"upstream_id":7365755140632559861,"namespace":"default","id":"818e736f-628e-43d2-a8ae-055cb251e392","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:32:09.25509772+08:00","start_ts":449570019540992002,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570019540992002,"checkpoint_ts":449570019540992002,"checkpoint_time":"2024-05-06 13:32:04.250"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/06 13:32:09.636 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] [2024/05/06 13:32:10.032 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark17000] [2024/05/06 13:32:10.032 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark17000] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [2024/05/06 13:32:10.129 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] [2024/05/06 13:32:10.146 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark14000] [ts=449570021044125703] [2024/05/06 13:32:10.146 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark14000] [2024/05/06 13:32:10.146 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570021044125703] [2024/05/06 13:32:10.148 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:32:10.151 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:32:10.155 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:32:10.159 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] + set +x < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:32:10 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/034eeb52-3350-420f-9d96-00214a4c2f03 {"id":"034eeb52-3350-420f-9d96-00214a4c2f03","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973527} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c631299cd 034eeb52-3350-420f-9d96-00214a4c2f03 /tidb/cdc/default/default/upstream/7365755152525734174 {"id":7365755152525734174,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/034eeb52-3350-420f-9d96-00214a4c2f03 {"id":"034eeb52-3350-420f-9d96-00214a4c2f03","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973527} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c631299cd 034eeb52-3350-420f-9d96-00214a4c2f03 /tidb/cdc/default/default/upstream/7365755152525734174 {"id":7365755152525734174,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/034eeb52-3350-420f-9d96-00214a4c2f03 {"id":"034eeb52-3350-420f-9d96-00214a4c2f03","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973527} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c631299cd 034eeb52-3350-420f-9d96-00214a4c2f03 /tidb/cdc/default/default/upstream/7365755152525734174 {"id":7365755152525734174,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/06 13:32:10.424 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] start tidb cluster in /tmp/tidb_cdc_test/multi_rocks Starting Upstream PD... + set +x ***************** properties ***************** "mysql.user"="root" "threadcount"="10" "operationcount"="0" "mysql.db"="consistent_replicate_nfs" "workload"="core" "readallfields"="true" "fieldcount"="3" "mysql.port"="4000" "dotransactions"="false" "updateproportion"="0" "requestdistribution"="uniform" "recordcount"="50000" "scanproportion"="0" "insertproportion"="0" "readproportion"="0" "mysql.host"="127.0.0.1" ********************************************** Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 24-th check, retry later table resourcecontrol.finish_mark not exists for 1-th check, retry later [2024/05/06 13:32:10.202 +08:00] [INFO] [main.go:86] ["running ddl test: 3 addDropColumnDDL2"] [2024/05/06 13:32:10.417 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 13:32:10.427 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 13:32:10.592 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 13:32:10.593 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 13:32:10.604 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 13:32:10.605 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 13:32:10.768 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 13:32:10.789 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/06 13:32:10.945 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 13:32:10.947 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 13:32:10.994 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 13:32:10.995 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 13:32:11.127 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/06 13:32:11.181 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 13:32:11.315 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 13:32:11.316 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/06 13:32:11.356 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 13:32:11.357 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318d3e540005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5gqnj-hmhkq-wjjv4, pid:4503, start at 2024-05-06 13:32:07.961793238 +0800 CST m=+5.281762565 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:07.973 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:08.008 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:08.008 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318d3e540005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5gqnj-hmhkq-wjjv4, pid:4503, start at 2024-05-06 13:32:07.961793238 +0800 CST m=+5.281762565 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:07.973 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:08.008 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:08.008 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318d3fd00010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-5gqnj-hmhkq-wjjv4, pid:4575, start at 2024-05-06 13:32:08.071669896 +0800 CST m=+5.322409408 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:08.078 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:08.052 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:08.052 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 3-th check, retry later [2024/05/06 13:32:11.499 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 13:32:11.538 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 13:32:11.671 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 13:32:11.673 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 13:32:11.706 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 13:32:11.707 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 13:32:11.840 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:32:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031d6ab0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570019032694800 MinTableBarrierTs:449570019294838800 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68 {"id":"7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973525} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4239 7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570019032694800,"min-table-barrier-ts":449570019294838800,"admin-job-type":0} /tidb/cdc/default/default/task/position/7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031d6ab0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570019032694800 MinTableBarrierTs:449570019294838800 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68 {"id":"7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973525} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4239 7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570019032694800,"min-table-barrier-ts":449570019294838800,"admin-job-type":0} /tidb/cdc/default/default/task/position/7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 + grep -q 'failed to get info:' {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031d6ab0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570019032694800 MinTableBarrierTs:449570019294838800 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68 {"id":"7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973525} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4239 7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570019032694800,"min-table-barrier-ts":449570019294838800,"admin-job-type":0} /tidb/cdc/default/default/task/position/7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/06 13:32:11.650 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] [2024/05/06 13:32:11.881 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/06 13:32:12.032 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 13:32:12.034 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/06 13:32:12.069 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/06 13:32:12.070 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 13:32:12.033 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] Verifying downstream PD is started... [2024/05/06 13:32:12.217 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/06 13:32:12.244 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/06 13:32:12.438 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] [2024/05/06 13:32:12.408 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 13:32:12.410 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/06 13:32:12.416 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/06 13:32:12.417 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 13:32:12.597 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/06 13:32:12.597 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/06 13:32:12.835 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/06 13:32:12.836 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 13:32:12.839 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 13:32:12.841 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later check data successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:32:10 CST 2024] <<<<<< run test case ddl_puller_lag success! >>>>>> [2024/05/06 13:32:13.023 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/06 13:32:13.046 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] table resourcecontrol.finish_mark exists [Mon May 6 13:32:13 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=return(true);github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=return(true)' + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.58685870.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 "is-owner": true, run task successfully table test.finish_mark not exists for 4-th check, retry later [2024/05/06 13:32:13.333 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark18000] [2024/05/06 13:32:13.333 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark18000] [2024/05/06 13:32:13.205 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/06 13:32:13.206 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/06 13:32:13.235 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/06 13:32:13.237 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/06 13:32:13.385 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] check diff successfully table mark.finish_mark_3 not exists for 25-th check, retry later [2024/05/06 13:32:13.419 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/06 13:32:13.552 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/06 13:32:13.553 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/06 13:32:13.605 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/06 13:32:13.607 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13285.out cli unsafe resolve-lock --region=52 [2024/05/06 13:32:13.730 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/06 13:32:13.792 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/06 13:32:13.733 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_resume_with_checkpoint_ts/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [2024/05/06 13:32:13.908 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/06 13:32:13.909 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/06 13:32:13.975 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/06 13:32:13.977 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/06 13:32:14.100 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/06 13:32:14.155 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/06 13:32:14.121 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] wait process cdc.test exit for 2-th time... [2024/05/06 13:32:14.297 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/06 13:32:14.298 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/06 13:32:14.329 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/06 13:32:14.330 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] owner pid: 4430 owner id 7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68 [Mon May 6 13:32:14 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.45584560.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 13:32:14.486 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/06 13:32:14.507 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/06 13:32:14.521 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] [2024/05/06 13:32:14.627 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark15000] [ts=449570021997543429] [2024/05/06 13:32:14.627 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark15000] [2024/05/06 13:32:14.627 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570021997543429] [2024/05/06 13:32:14.629 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:32:14.633 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:32:14.725 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:32:14.730 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:32:14 CST 2024] <<<<<< run test case resourcecontrol success! >>>>>> [2024/05/06 13:32:14.669 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/06 13:32:14.670 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/06 13:32:14.679 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/06 13:32:14.681 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/05/06 13:32:14.848 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/05/06 13:32:14.849 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/05/06 13:32:14.730 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/06 13:32:14.732 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/06 13:32:14.733 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811] [2024/05/06 13:32:14.733 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/06 13:32:14.734 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/06 13:32:14.901 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=9cbe3e2c-4ad8-4840-8911-e0dbef1c4359] [captureID=3398e8ae-814f-4a43-9659-7de7839827c5] [processorDetail="{\"table_ids\":[]}"] [2024/05/06 13:32:15.029 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/05/06 13:32:15.030 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/05/06 13:32:15.030 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/05/06 13:32:15.032 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.finish_mark not exists for 5-th check, retry later [2024/05/06 13:32:15.101 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=9cbe3e2c-4ad8-4840-8911-e0dbef1c4359] [captureID=7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811] [processorDetail="{\"table_ids\":[106,108]}"] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13320.out cli unsafe resolve-lock --region=52 --ts=449570021155799043 [2024/05/06 13:32:15.200 +08:00] [INFO] [main.go:220] ["1 insert success: 2700"] [2024/05/06 13:32:15.301 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=9cbe3e2c-4ad8-4840-8911-e0dbef1c4359] [captureID=e6d591a2-27ae-435c-8275-1befd2fa5869] [processorDetail="{\"table_ids\":[]}"] [2024/05/06 13:32:15.302 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"3398e8ae-814f-4a43-9659-7de7839827c5\":[],\"7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811\":[{\"ID\":106,\"Changefeed\":\"9cbe3e2c-4ad8-4840-8911-e0dbef1c4359\"},{\"ID\":108,\"Changefeed\":\"9cbe3e2c-4ad8-4840-8911-e0dbef1c4359\"}],\"e6d591a2-27ae-435c-8275-1befd2fa5869\":[]}"] [2024/05/06 13:32:15.302 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=9cbe3e2c-4ad8-4840-8911-e0dbef1c4359&target-cp-id=3398e8ae-814f-4a43-9659-7de7839827c5&table-id=106"] [2024/05/06 13:32:15.352 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=106] [2024/05/06 13:32:15.352 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=9cbe3e2c-4ad8-4840-8911-e0dbef1c4359&target-cp-id=3398e8ae-814f-4a43-9659-7de7839827c5&table-id=108"] [2024/05/06 13:32:15.401 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=108] [2024/05/06 13:32:15.401 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811] [targetCapture=3398e8ae-814f-4a43-9659-7de7839827c5] table move_table.check1 exists table mark.finish_mark_3 not exists for 26-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used PASS [2024/05/06 13:32:15.333 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] check diff successfully [2024/05/06 13:32:10.350 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 2 another is 5"] start tidb cluster in /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... coverage: 2.0% of statements in github.com/pingcap/tiflow/... [2024/05/06 13:32:15.748 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:32:15 CST 2024] <<<<<< run test case kafka_big_messages success! >>>>>> [2024/05/06 13:32:16.222 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:32:16 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a795d54d-c08a-462c-b3f1-12de84d4dae4 {"id":"a795d54d-c08a-462c-b3f1-12de84d4dae4","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973533} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c632308d0 a795d54d-c08a-462c-b3f1-12de84d4dae4 /tidb/cdc/default/default/upstream/7365755171986536853 {"id":7365755171986536853,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a795d54d-c08a-462c-b3f1-12de84d4dae4 {"id":"a795d54d-c08a-462c-b3f1-12de84d4dae4","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973533} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c632308d0 a795d54d-c08a-462c-b3f1-12de84d4dae4 /tidb/cdc/default/default/upstream/7365755171986536853 {"id":7365755171986536853,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a795d54d-c08a-462c-b3f1-12de84d4dae4 {"id":"a795d54d-c08a-462c-b3f1-12de84d4dae4","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973533} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c632308d0 a795d54d-c08a-462c-b3f1-12de84d4dae4 /tidb/cdc/default/default/upstream/7365755171986536853 {"id":7365755171986536853,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.cli.5919.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 table processor_delay.t1 exists table processor_delay.t2 exists table processor_delay.t3 exists table processor_delay.t4 exists table processor_delay.t5 exists table processor_delay.t6 exists table processor_delay.t7 exists table processor_delay.t8 exists table processor_delay.t9 exists table processor_delay.t10 exists table processor_delay.t11 exists table processor_delay.t12 exists table processor_delay.t13 not exists for 1-th check, retry later table processor_delay.t13 exists table processor_delay.t14 not exists for 1-th check, retry later [2024/05/06 13:32:16.632 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark19000] [2024/05/06 13:32:16.632 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark19000] Create changefeed successfully! ID: 61325ec1-ebb9-4bca-9809-0934957ec8f2 Info: {"upstream_id":7365755171986536853,"namespace":"default","id":"61325ec1-ebb9-4bca-9809-0934957ec8f2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:32:16.538688769+08:00","start_ts":449570022741245956,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570022741245956,"checkpoint_ts":449570022741245956,"checkpoint_time":"2024-05-06 13:32:16.458"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... table test.finish_mark not exists for 6-th check, retry later Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 27-th check, retry later [2024/05/06 13:32:17.349 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 139k 4029 --:--:-- --:--:-- --:--:-- 207k { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:32:17 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0039499e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570022715817991 MinTableBarrierTs:449570022978224137 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c519e6f-945c-426c-8f53-bc1805c4bce8 {"id":"5c519e6f-945c-426c-8f53-bc1805c4bce8","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973534} /tidb/cdc/default/__cdc_meta__/capture/7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68 {"id":"7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973525} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4239 7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac429a 5c519e6f-945c-426c-8f53-bc1805c4bce8 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570022715817991,"min-table-barrier-ts":449570022978224137,"admin-job-type":0} /tidb/cdc/default/default/task/position/5c519e6f-945c-426c-8f53-bc1805c4bce8/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0039499e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570022715817991 MinTableBarrierTs:449570022978224137 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c519e6f-945c-426c-8f53-bc1805c4bce8 {"id":"5c519e6f-945c-426c-8f53-bc1805c4bce8","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973534} /tidb/cdc/default/__cdc_meta__/capture/7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68 {"id":"7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973525} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4239 7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac429a 5c519e6f-945c-426c-8f53-bc1805c4bce8 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570022715817991,"min-table-barrier-ts":449570022978224137,"admin-job-type":0} /tidb/cdc/default/default/task/position/5c519e6f-945c-426c-8f53-bc1805c4bce8/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0039499e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570022715817991 MinTableBarrierTs:449570022978224137 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c519e6f-945c-426c-8f53-bc1805c4bce8 {"id":"5c519e6f-945c-426c-8f53-bc1805c4bce8","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973534} /tidb/cdc/default/__cdc_meta__/capture/7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68 {"id":"7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973525} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4239 7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac429a 5c519e6f-945c-426c-8f53-bc1805c4bce8 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570022715817991,"min-table-barrier-ts":449570022978224137,"admin-job-type":0} /tidb/cdc/default/default/task/position/5c519e6f-945c-426c-8f53-bc1805c4bce8/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "7b4bf3af-c782-4a2c-95a0-f1e5d9eb3c68" | grep id [2024/05/06 13:32:17.832 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] table processor_delay.t14 exists table processor_delay.t15 not exists for 1-th check, retry later + set +x [2024/05/06 13:32:18.025 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark16000] [ts=449570022836404229] [2024/05/06 13:32:18.025 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark16000] [2024/05/06 13:32:18.026 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570022836404229] [2024/05/06 13:32:18.029 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:32:18.032 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:32:18.037 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:32:18.121 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:32:18.236 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] [2024/05/06 13:32:18.632 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark19000] table test.finish_mark not exists for 7-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "cluster-id": "default" "id": "5c519e6f-945c-426c-8f53-bc1805c4bce8", "cluster-id": "default" run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 28-th check, retry later cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table processor_resolved_ts_fallback.t1 exists [Mon May 6 13:32:19 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.59975999.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark not exists for 1-th check, retry later [2024/05/06 13:32:19.526 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:32:18 CST 2024] <<<<<< run test case kafka_compression success! >>>>>> table ddl_attributes.finish_mark not exists for 2-th check, retry later [2024/05/06 13:32:19.823 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20001] [2024/05/06 13:32:19.823 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] [2024/05/06 13:32:19.824 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20002] [2024/05/06 13:32:19.824 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] [2024/05/06 13:32:19.825 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20003] [2024/05/06 13:32:19.825 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20004] table processor_delay.t15 exists table processor_delay.t16 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 13:32:19.900 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark20000] [2024/05/06 13:32:19.900 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20004] [2024/05/06 13:32:20.329 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] capture_id: 5c519e6f-945c-426c-8f53-bc1805c4bce8 [2024/05/06 13:32:20.425 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark17000] [ts=449570023665827845] [2024/05/06 13:32:20.425 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark17000] [2024/05/06 13:32:20.425 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570023665827845] [2024/05/06 13:32:20.428 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:32:20.430 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:32:20.435 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:32:20.439 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:32:20.439 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark20000] [2024/05/06 13:32:20.462 +08:00] [INFO] [main.go:86] ["running ddl test: 4 modifyColumnDDL"] [2024/05/06 13:32:20.666 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 13:32:20.669 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 13:32:20.721 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark19000] [2024/05/06 13:32:20.847 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: mysql... <<================= } % Total % Received % Xferd Average Speed Time Time Time Current [2024/05/06 13:32:20.802 +08:00] [INFO] [main.go:61] ["table mover started"] table test.finish_mark not exists for 8-th check, retry later [2024/05/06 13:32:20.805 +08:00] [INFO] [main.go:166] ["new cluster initialized"] Dload Upload Total Spent Left Speed [2024/05/06 13:32:20.806 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811] The 1 times to try to start tidb cluster... [2024/05/06 13:32:20.849 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 13:32:20.806 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 214k 0 --:--:-- --:--:-- --:--:-- 177k { [2024/05/06 13:32:20.850 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] "version": "v8.2.0-alpha-22-gd72d260c0", [2024/05/06 13:32:20.852 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 13:32:20.806 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] "git_hash": "d72d260c017170db94deda602246fcddfea2a4a1", "id": "5e1c2b1e-2276-48a2-ba8d-41bdf2558aed", "pid": 12290, "is_owner": true }wait process cdc.test exit for 1-th time... table mark.finish_mark_3 not exists for 29-th check, retry later [2024/05/06 13:32:21.025 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/06 13:32:21.027 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 13:32:21.198 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 13:32:21.199 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 13:32:21.209 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 13:32:21.211 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 13:32:21.366 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/06 13:32:21.386 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 13:32:21.001 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=9cbe3e2c-4ad8-4840-8911-e0dbef1c4359] [captureID=3398e8ae-814f-4a43-9659-7de7839827c5] [processorDetail="{\"table_ids\":[108,110]}"] [2024/05/06 13:32:21.201 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=9cbe3e2c-4ad8-4840-8911-e0dbef1c4359] [captureID=7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811] [processorDetail="{\"table_ids\":[]}"] [2024/05/06 13:32:21.401 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=9cbe3e2c-4ad8-4840-8911-e0dbef1c4359] [captureID=e6d591a2-27ae-435c-8275-1befd2fa5869] [processorDetail="{\"table_ids\":[]}"] [2024/05/06 13:32:21.401 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"3398e8ae-814f-4a43-9659-7de7839827c5\":[{\"ID\":108,\"Changefeed\":\"9cbe3e2c-4ad8-4840-8911-e0dbef1c4359\"},{\"ID\":110,\"Changefeed\":\"9cbe3e2c-4ad8-4840-8911-e0dbef1c4359\"}],\"7cc73fc1-1f0c-45bc-b68d-edd7a2a6d811\":[],\"e6d591a2-27ae-435c-8275-1befd2fa5869\":[]}"] [2024/05/06 13:32:21.401 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=9cbe3e2c-4ad8-4840-8911-e0dbef1c4359&target-cp-id=e6d591a2-27ae-435c-8275-1befd2fa5869&table-id=108"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:32:21.538 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 13:32:21.540 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/06 13:32:21.573 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 13:32:21.575 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 13:32:21.451 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=108] [2024/05/06 13:32:21.451 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=9cbe3e2c-4ad8-4840-8911-e0dbef1c4359&target-cp-id=e6d591a2-27ae-435c-8275-1befd2fa5869&table-id=110"] [2024/05/06 13:32:21.501 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=110] [2024/05/06 13:32:21.501 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=3398e8ae-814f-4a43-9659-7de7839827c5] [targetCapture=e6d591a2-27ae-435c-8275-1befd2fa5869] check diff successfully wait process cdc.test exit for 2-th time... table processor_delay.t16 exists table processor_delay.t17 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:32:21.540 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2024/05/06 13:32:21.714 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 13:32:21.802 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:32:21.946 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 13:32:21.948 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 13:32:21.993 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 13:32:21.995 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 13:32:22.119 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 13:32:22.163 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] table move_table.check2 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:32:22 CST 2024] <<<<<< run test case cli_with_auth success! >>>>>> table ddl_attributes.finish_mark not exists for 3-th check, retry later [2024/05/06 13:32:22.295 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 13:32:22.297 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/06 13:32:22.342 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 13:32:22.345 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/06 13:32:22.439 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark20000] [2024/05/06 13:32:22.467 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/06 13:32:22.522 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/06 13:32:22.655 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 13:32:22.657 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/06 13:32:22.711 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 13:32:22.713 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:32:22 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/61325ec1-ebb9-4bca-9809-0934957ec8f2 {UpstreamID:7365755171986536853 Namespace:default ID:61325ec1-ebb9-4bca-9809-0934957ec8f2 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:32:16.538688769 +0800 CST StartTs:449570022741245956 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003fd2cf0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570022754091014} {CheckpointTs:449570024130347017 MinTableBarrierTs:449570024130347017 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a795d54d-c08a-462c-b3f1-12de84d4dae4 {"id":"a795d54d-c08a-462c-b3f1-12de84d4dae4","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973533} /tidb/cdc/default/__cdc_meta__/capture/bf9fa64b-f311-4ad0-bd61-b6942be93c77 {"id":"bf9fa64b-f311-4ad0-bd61-b6942be93c77","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973539} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c632308d0 a795d54d-c08a-462c-b3f1-12de84d4dae4 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63230949 bf9fa64b-f311-4ad0-bd61-b6942be93c77 /tidb/cdc/default/default/changefeed/info/61325ec1-ebb9-4bca-9809-0934957ec8f2 {"upstream-id":7365755171986536853,"namespace":"default","changefeed-id":"61325ec1-ebb9-4bca-9809-0934957ec8f2","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:32:16.538688769+08:00","start-ts":449570022741245956,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570022754091014} /tidb/cdc/default/default/changefeed/status/61325ec1-ebb9-4bca-9809-0934957ec8f2 {"checkpoint-ts":449570024130347017,"min-table-barrier-ts":449570024130347017,"admin-job-type":0} /tidb/cdc/default/default/task/position/a795d54d-c08a-462c-b3f1-12de84d4dae4/61325ec1-ebb9-4bca-9809-0934957ec8f2 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/bf9fa64b-f311-4ad0-bd61-b6942be93c77/61325ec1-ebb9-4bca-9809-0934957ec8f2 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755171986536853 {"id":7365755171986536853,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/61325ec1-ebb9-4bca-9809-0934957ec8f2 {UpstreamID:7365755171986536853 Namespace:default ID:61325ec1-ebb9-4bca-9809-0934957ec8f2 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:32:16.538688769 +0800 CST StartTs:449570022741245956 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003fd2cf0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570022754091014} {CheckpointTs:449570024130347017 MinTableBarrierTs:449570024130347017 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a795d54d-c08a-462c-b3f1-12de84d4dae4 {"id":"a795d54d-c08a-462c-b3f1-12de84d4dae4","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973533} /tidb/cdc/default/__cdc_meta__/capture/bf9fa64b-f311-4ad0-bd61-b6942be93c77 {"id":"bf9fa64b-f311-4ad0-bd61-b6942be93c77","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973539} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c632308d0 a795d54d-c08a-462c-b3f1-12de84d4dae4 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63230949 bf9fa64b-f311-4ad0-bd61-b6942be93c77 /tidb/cdc/default/default/changefeed/info/61325ec1-ebb9-4bca-9809-0934957ec8f2 {"upstream-id":7365755171986536853,"namespace":"default","changefeed-id":"61325ec1-ebb9-4bca-9809-0934957ec8f2","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:32:16.538688769+08:00","start-ts":449570022741245956,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570022754091014} /tidb/cdc/default/default/changefeed/status/61325ec1-ebb9-4bca-9809-0934957ec8f2 {"checkpoint-ts":449570024130347017,"min-table-barrier-ts":449570024130347017,"admin-job-type":0} /tidb/cdc/default/default/task/position/a795d54d-c08a-462c-b3f1-12de84d4dae4/61325ec1-ebb9-4bca-9809-0934957ec8f2 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/bf9fa64b-f311-4ad0-bd61-b6942be93c77/61325ec1-ebb9-4bca-9809-0934957ec8f2 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755171986536853 {"id":7365755171986536853,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/61325ec1-ebb9-4bca-9809-0934957ec8f2 {UpstreamID:7365755171986536853 Namespace:default ID:61325ec1-ebb9-4bca-9809-0934957ec8f2 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:32:16.538688769 +0800 CST StartTs:449570022741245956 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003fd2cf0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570022754091014} {CheckpointTs:449570024130347017 MinTableBarrierTs:449570024130347017 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a795d54d-c08a-462c-b3f1-12de84d4dae4 {"id":"a795d54d-c08a-462c-b3f1-12de84d4dae4","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973533} /tidb/cdc/default/__cdc_meta__/capture/bf9fa64b-f311-4ad0-bd61-b6942be93c77 {"id":"bf9fa64b-f311-4ad0-bd61-b6942be93c77","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973539} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c632308d0 a795d54d-c08a-462c-b3f1-12de84d4dae4 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63230949 bf9fa64b-f311-4ad0-bd61-b6942be93c77 /tidb/cdc/default/default/changefeed/info/61325ec1-ebb9-4bca-9809-0934957ec8f2 {"upstream-id":7365755171986536853,"namespace":"default","changefeed-id":"61325ec1-ebb9-4bca-9809-0934957ec8f2","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:32:16.538688769+08:00","start-ts":449570022741245956,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570022754091014} /tidb/cdc/default/default/changefeed/status/61325ec1-ebb9-4bca-9809-0934957ec8f2 {"checkpoint-ts":449570024130347017,"min-table-barrier-ts":449570024130347017,"admin-job-type":0} /tidb/cdc/default/default/task/position/a795d54d-c08a-462c-b3f1-12de84d4dae4/61325ec1-ebb9-4bca-9809-0934957ec8f2 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/bf9fa64b-f311-4ad0-bd61-b6942be93c77/61325ec1-ebb9-4bca-9809-0934957ec8f2 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755171986536853 {"id":7365755171986536853,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:32:22 CST 2024] <<<<<< run test case kafka_messages success! >>>>>> INSERT - Takes(s): 10.0, Count: 26475, OPS: 2649.1, Avg(us): 3735, Min(us): 913, Max(us): 176797, 95th(us): 4000, 99th(us): 84000 [2024/05/06 13:32:22.734 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark19000] [2024/05/06 13:32:22.847 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/06 13:32:22.906 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] table test.finish_mark not exists for 9-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:32:23.047 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 13:32:23.049 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/06 13:32:23.100 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 13:32:23.102 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/06 13:32:23.241 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:32:23.291 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/06 13:32:23.425 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/06 13:32:23.427 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/06 13:32:23.474 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/06 13:32:23.476 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] table processor_resolved_ts_fallback.t2 exists table processor_resolved_ts_fallback.t3 not exists for 1-th check, retry later table processor_delay.t17 exists table mark.finish_mark_3 not exists for 30-th check, retry later [2024/05/06 13:32:23.625 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/06 13:32:23.657 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/06 13:32:23.552 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] [2024/05/06 13:32:23.565 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark18000] [ts=449570024334032901] [2024/05/06 13:32:23.565 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark18000] [2024/05/06 13:32:23.565 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570024334032901] [2024/05/06 13:32:23.567 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:32:23.569 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:32:23.574 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:32:23.577 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:32:23.577 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] table processor_delay.t18 not exists for 1-th check, retry later [2024/05/06 13:32:23.869 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/06 13:32:23.871 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/06 13:32:23.901 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/06 13:32:23.903 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_attributes.finish_mark not exists for 4-th check, retry later [2024/05/06 13:32:24.061 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/06 13:32:24.099 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/06 13:32:24.250 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/06 13:32:24.252 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... [2024/05/06 13:32:24.451 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark20000] [2024/05/06 13:32:24.282 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/06 13:32:24.284 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/06 13:32:24.452 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/06 13:32:24.481 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table move_table.check2 exists check diff successfully \033[0;36m<<< Run all test success >>>\033[0m [2024/05/06 13:32:24.630 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/06 13:32:24.631 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/06 13:32:24.674 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/06 13:32:24.676 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] check_changefeed_state http://127.0.0.1:2379 4ab67143-af55-429a-92d5-f6b3d2b118e0 finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=4ab67143-af55-429a-92d5-f6b3d2b118e0 + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 4ab67143-af55-429a-92d5-f6b3d2b118e0 -s + info='{ "upstream_id": 7365754773601530271, "namespace": "default", "id": "4ab67143-af55-429a-92d5-f6b3d2b118e0", "state": "finished", "checkpoint_tso": 449570021691097090, "checkpoint_time": "2024-05-06 13:32:12.452", "error": null }' + echo '{ "upstream_id": 7365754773601530271, "namespace": "default", "id": "4ab67143-af55-429a-92d5-f6b3d2b118e0", "state": "finished", "checkpoint_tso": 449570021691097090, "checkpoint_time": "2024-05-06 13:32:12.452", "error": null }' { "upstream_id": 7365754773601530271, "namespace": "default", "id": "4ab67143-af55-429a-92d5-f6b3d2b118e0", "state": "finished", "checkpoint_tso": 449570021691097090, "checkpoint_time": "2024-05-06 13:32:12.452", "error": null } ++ echo '{' '"upstream_id":' 7365754773601530271, '"namespace":' '"default",' '"id":' '"4ab67143-af55-429a-92d5-f6b3d2b118e0",' '"state":' '"finished",' '"checkpoint_tso":' 449570021691097090, '"checkpoint_time":' '"2024-05-06' '13:32:12.452",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7365754773601530271, '"namespace":' '"default",' '"id":' '"4ab67143-af55-429a-92d5-f6b3d2b118e0",' '"state":' '"finished",' '"checkpoint_tso":' 449570021691097090, '"checkpoint_time":' '"2024-05-06' '13:32:12.452",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:32:20 CST 2024] <<<<<< run test case changefeed_finish success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc already exists) table test.finish_mark not exists for 10-th check, retry later [2024/05/06 13:32:24.746 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark19000] [2024/05/06 13:32:24.759 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark19000] [ts=449570024858583045] [2024/05/06 13:32:24.759 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark19000] [2024/05/06 13:32:24.759 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570024858583045] [2024/05/06 13:32:24.761 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:32:24.764 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:32:24.768 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:32:24.772 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:32:24.772 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] [Pipeline] // cache [2024/05/06 13:32:24.818 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/06 13:32:24.872 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/06 13:32:25.008 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/06 13:32:25.010 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [Pipeline] } [2024/05/06 13:32:25.058 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/06 13:32:25.060 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/06 13:32:25.202 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/05/06 13:32:25.246 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] wait process cdc.test exit for 1-th time... [Pipeline] // dir ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // withCredentials table processor_resolved_ts_fallback.t3 exists cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^2$' [Pipeline] } [2024/05/06 13:32:25.397 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/05/06 13:32:25.400 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] [2024/05/06 13:32:25.440 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/05/06 13:32:25.442 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] table mark.finish_mark_3 not exists for 31-th check, retry later [Pipeline] // timeout VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318e22200007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-st1k9-fprpv-trhhs, pid:6480, start at 2024-05-06 13:32:22.542757357 +0800 CST m=+5.465634728 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:22.549 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:22.536 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:22.536 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318e22200007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-st1k9-fprpv-trhhs, pid:6480, start at 2024-05-06 13:32:22.542757357 +0800 CST m=+5.465634728 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:22.549 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:22.536 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:22.536 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318e22e00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-st1k9-fprpv-trhhs, pid:6549, start at 2024-05-06 13:32:22.630362204 +0800 CST m=+5.483037122 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:22.637 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:22.634 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:22.634 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/common_1/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/common_1/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/common_1/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [Pipeline] } 2 run task successfully table processor_delay.t18 exists table processor_delay.t19 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... [Pipeline] // stage [Pipeline] } table ddl_attributes.finish_mark not exists for 5-th check, retry later killing cdc1: 5873 cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 1-th time, retry later [Pipeline] // container =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:32:25 CST 2024] <<<<<< run test case kafka_sink_error_resume success! >>>>>> [Pipeline] } [Pipeline] // withEnv [Pipeline] } wait process cdc.test exit for 3-th time... [Pipeline] // node [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [2024/05/06 13:32:26.464 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark20000] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:32:26 CST 2024] <<<<<< run test case move_table success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 11-th check, retry later table mark.finish_mark_3 not exists for 32-th check, retry later table ddl_attributes.finish_mark not exists for 6-th check, retry later cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 2-th time, retry later nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table processor_delay.t19 exists table processor_delay.t20 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318e68840017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-r8vbf-lj78v-9fw4x, pid:6683, start at 2024-05-06 13:32:27.069075804 +0800 CST m=+5.173488897 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:27.075 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:27.041 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:27.041 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318e68840017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-r8vbf-lj78v-9fw4x, pid:6683, start at 2024-05-06 13:32:27.069075804 +0800 CST m=+5.173488897 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:27.075 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:27.041 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:27.041 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318e6ad00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-r8vbf-lj78v-9fw4x, pid:6773, start at 2024-05-06 13:32:27.194957755 +0800 CST m=+5.241127697 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:27.204 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:27.188 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:27.188 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 13:32:28.476 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark20000] [2024/05/06 13:32:28.491 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark20000] [ts=449570025382871045] [2024/05/06 13:32:28.491 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark20000] [2024/05/06 13:32:28.491 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449570025382871045] [2024/05/06 13:32:28.495 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:32:28.498 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:32:28.502 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 13:32:28.506 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 13:32:28.506 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] [2024/05/06 13:32:28.506 +08:00] [INFO] [case.go:615] ["bank test finished"] [valid=20] [tries=20] [ratio=1] + cleanup_process cdc.test Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 12-th check, retry later wait process cdc.test exit for 1-th time... table mark.finish_mark_3 not exists for 33-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:32:28 CST 2024] <<<<<< run test case mq_sink_lost_callback success! >>>>>> wait process cdc.test exit for 2-th time... table processor_delay.t20 exists + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.7959.out cli tso query --pd=http://127.0.0.1:2379 table processor_delay.t21 not exists for 1-th check, retry later empty select id, val from test.availability1 where id=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later wait process cdc.test exit for 3-th time... [Mon May 6 13:32:30 CST 2024] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.80158017.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit ++ date + echo '[Mon May 6 13:32:30 CST 2024] <<<<<< run test case bank success! >>>>>>' [Mon May 6 13:32:30 CST 2024] <<<<<< run test case bank success! >>>>>> + stop_tidb_cluster [2024/05/06 13:32:30.714 +08:00] [INFO] [main.go:86] ["running ddl test: 5 addDropIndexDDL"] INSERT - Takes(s): 20.0, Count: 36424, OPS: 1821.7, Avg(us): 5463, Min(us): 913, Max(us): 176797, 95th(us): 23000, 99th(us): 88000 [2024/05/06 13:32:30.917 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 13:32:30.920 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] table test.finish_mark not exists for 13-th check, retry later table mark.finish_mark_3 not exists for 34-th check, retry later [2024/05/06 13:32:31.087 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 13:32:31.088 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 13:32:31.089 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 13:32:31.089 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 13:32:31.266 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 13:32:31.279 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_attributes.finish_mark exists check diff successfully [2024/05/06 13:32:31.505 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 13:32:31.507 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 13:32:31.518 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 13:32:31.520 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449570026235101185 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570026235101185 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:32:31 CST 2024] <<<<<< START cdc server in common_1 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.79998001.out server --log-file /tmp/tidb_cdc_test/common_1/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/common_1/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 13:32:31.792 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 13:32:31.793 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] table processor_delay.t21 exists table processor_delay.t22 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... nonempty select id, val from test.availability2 where id=1 and val=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/06 13:32:32.029 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 13:32:32.032 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 13:32:32.032 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/06 13:32:32.033 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] wait process cdc.test exit for 2-th time... [2024/05/06 13:32:32.283 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 13:32:32.284 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' [2024/05/06 13:32:32.541 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 13:32:32.542 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 3-th time, retry later wait process cdc.test exit for 3-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:32:32 CST 2024] <<<<<< run test case mq_sink_dispatcher success! >>>>>> [2024/05/06 13:32:32.571 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 13:32:32.574 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 13:32:32.865 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 13:32:32.878 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] table test.finish_mark not exists for 14-th check, retry later table mark.finish_mark_3 not exists for 35-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:32:33.170 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/06 13:32:33.172 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 13:32:33.192 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 13:32:33.196 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:32:33 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7aad713f-c747-4b97-abba-acd57c36d154 {"id":"7aad713f-c747-4b97-abba-acd57c36d154","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973550} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c636e2ac3 7aad713f-c747-4b97-abba-acd57c36d154 /tidb/cdc/default/default/upstream/7365755260047450314 {"id":7365755260047450314,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7aad713f-c747-4b97-abba-acd57c36d154 {"id":"7aad713f-c747-4b97-abba-acd57c36d154","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973550} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c636e2ac3 7aad713f-c747-4b97-abba-acd57c36d154 /tidb/cdc/default/default/upstream/7365755260047450314 {"id":7365755260047450314,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7aad713f-c747-4b97-abba-acd57c36d154 {"id":"7aad713f-c747-4b97-abba-acd57c36d154","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973550} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c636e2ac3 7aad713f-c747-4b97-abba-acd57c36d154 /tidb/cdc/default/default/upstream/7365755260047450314 {"id":7365755260047450314,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/06 13:32:33.514 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/06 13:32:33.518 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [Mon May 6 13:32:33 CST 2024] <<<<<< run test case ddl_attributes success! >>>>>> table processor_delay.t22 exists table processor_delay.t23 not exists for 1-th check, retry later table test.table1 not exists for 1-th check, retry later [2024/05/06 13:32:33.834 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 13:32:33.838 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/06 13:32:33.847 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/06 13:32:33.848 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [2024/05/06 13:32:34.140 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/06 13:32:34.193 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... nonempty select id, val from test.availability2 where id=1 and val=22 run task successfully empty select id, val from test.availability2 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:32:34 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eeaf66f3-915b-461d-abcd-cd8c7a2ace00 {"id":"eeaf66f3-915b-461d-abcd-cd8c7a2ace00","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973551} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c635d58dc eeaf66f3-915b-461d-abcd-cd8c7a2ace00 /tidb/cdc/default/default/upstream/7365755240474532652 {"id":7365755240474532652,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eeaf66f3-915b-461d-abcd-cd8c7a2ace00 {"id":"eeaf66f3-915b-461d-abcd-cd8c7a2ace00","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973551} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c635d58dc eeaf66f3-915b-461d-abcd-cd8c7a2ace00 /tidb/cdc/default/default/upstream/7365755240474532652 {"id":7365755240474532652,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eeaf66f3-915b-461d-abcd-cd8c7a2ace00 {"id":"eeaf66f3-915b-461d-abcd-cd8c7a2ace00","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973551} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c635d58dc eeaf66f3-915b-461d-abcd-cd8c7a2ace00 /tidb/cdc/default/default/upstream/7365755240474532652 {"id":7365755240474532652,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: b21137e9-b788-42df-b4d8-fc843c92725d Info: {"upstream_id":7365755240474532652,"namespace":"default","id":"b21137e9-b788-42df-b4d8-fc843c92725d","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-06T13:32:34.556877502+08:00","start_ts":449570026235101185,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570026235101185,"checkpoint_ts":449570026235101185,"checkpoint_time":"2024-05-06 13:32:29.786"} start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/06 13:32:34.469 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 13:32:34.473 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/06 13:32:34.531 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/06 13:32:34.533 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 13:32:34.809 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/06 13:32:34.883 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] table test.finish_mark not exists for 15-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 13:32:35.154 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/06 13:32:35.161 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/06 13:32:35.237 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/06 13:32:35.238 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] table mark.finish_mark_3 not exists for 36-th check, retry later [2024/05/06 13:32:35.549 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/06 13:32:35.643 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.table1 exists table test.table2 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:32:35 CST 2024] <<<<<< run test case kafka_column_selector success! >>>>>> table processor_delay.t23 exists table processor_delay.t24 not exists for 1-th check, retry later empty select id, val from test.availability2 where id=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 16-th check, retry later table mark.finish_mark_3 not exists for 37-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.table2 exists Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 1-th time, retry later table processor_delay.t24 exists table processor_delay.t25 not exists for 1-th check, retry later nonempty select id, val from test.availability3 where id=1 and val=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_column_selector_avro/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:32:38 CST 2024] <<<<<< run test case kafka_column_selector_avro success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318ef05c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54, pid:6581, start at 2024-05-06 13:32:35.778661093 +0800 CST m=+5.204854275 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:35.787 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:35.785 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:35.785 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318ef05c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54, pid:6581, start at 2024-05-06 13:32:35.778661093 +0800 CST m=+5.204854275 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:35.787 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:35.785 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:35.785 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318eef740014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54, pid:6647, start at 2024-05-06 13:32:35.709213016 +0800 CST m=+5.078026107 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:35.715 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:35.677 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:35.677 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' 1 run task successfully check diff failed 1-th time, retry later table test.finish_mark not exists for 17-th check, retry later table mark.finish_mark_3 not exists for 38-th check, retry later table processor_delay.t25 exists table processor_delay.t26 not exists for 1-th check, retry later nonempty select id, val from test.availability3 where id=1 and val=22 run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) empty select id, val from test.availability3 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/kill_owner_with_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 13:32:40.905 +08:00] [INFO] [main.go:78] ["runDDLTest take 1m1.179364661s"] INSERT - Takes(s): 30.0, Count: 48675, OPS: 1622.8, Avg(us): 6138, Min(us): 913, Max(us): 186058, 95th(us): 64000, 99th(us): 88000 check diff successfully Resume changefeed with checkpointTs3 449570029080674307 [Mon May 6 13:32:40 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.78997901.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 18-th check, retry later table mark.finish_mark_3 not exists for 39-th check, retry later table test.table5 not exists for 1-th check, retry later table mark.finish_mark_0 exists table mark.finish_mark_1 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/autorandom/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/debezium/run.sh using Sink-Type: mysql... <<================= =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:32:41 CST 2024] <<<<<< run test case lossy_ddl success! >>>>>> Run finished, takes 31.090205879s INSERT - Takes(s): 31.1, Count: 50000, OPS: 1608.5, Avg(us): 6180, Min(us): 913, Max(us): 186058, 95th(us): 65000, 99th(us): 88000 table processor_delay.t26 exists table processor_delay.t27 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... table consistent_replicate_nfs.usertable exists Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) empty select id, val from test.availability3 where id=1 run task successfully test_gap_between_watch_capture pass wait process cdc.test exit for 3-th time... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:32:42 CST 2024] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> wait process cdc.test exit for 2-th time... table mark.finish_mark_3 not exists for 40-th check, retry later table test.finish_mark not exists for 19-th check, retry later table mark.finish_mark_1 not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m table processor_delay.t27 exists cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case delete_owner_key [Mon May 6 13:32:43 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.49834985.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.table5 not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318f5f480013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-dnsg9-wwwnp-xssfm, pid:4024, start at 2024-05-06 13:32:42.86213151 +0800 CST m=+5.141167914 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:42.868 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:42.834 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:42.834 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318f5f480013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-dnsg9-wwwnp-xssfm, pid:4024, start at 2024-05-06 13:32:42.86213151 +0800 CST m=+5.141167914 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:42.868 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:42.834 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:42.834 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318f60b80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-dnsg9-wwwnp-xssfm, pid:4112, start at 2024-05-06 13:32:42.95513068 +0800 CST m=+5.151899405 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:42.961 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:42.926 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:42.926 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table processor_delay.t28 not exists for 1-th check, retry later [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc already exists) table common_1.v1 exists table common_1.recover_and_insert not exists for 1-th check, retry later [Pipeline] // cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:32:43 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4686c0f6-5788-480f-ade5-bdafd9e70e12 {"id":"4686c0f6-5788-480f-ade5-bdafd9e70e12","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973561} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c639020cf 4686c0f6-5788-480f-ade5-bdafd9e70e12 /tidb/cdc/default/default/upstream/7365755301910702846 {"id":7365755301910702846,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4686c0f6-5788-480f-ade5-bdafd9e70e12 {"id":"4686c0f6-5788-480f-ade5-bdafd9e70e12","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973561} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c639020cf 4686c0f6-5788-480f-ade5-bdafd9e70e12 /tidb/cdc/default/default/upstream/7365755301910702846 {"id":7365755301910702846,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4686c0f6-5788-480f-ade5-bdafd9e70e12 {"id":"4686c0f6-5788-480f-ade5-bdafd9e70e12","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973561} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c639020cf 4686c0f6-5788-480f-ade5-bdafd9e70e12 /tidb/cdc/default/default/upstream/7365755301910702846 {"id":7365755301910702846,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.7957.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-only-block-related-table Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7365755301910702846,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-06T13:32:44.395107764+08:00","start_ts":449570030036189189,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570030036189189,"checkpoint_ts":449570030036189189,"checkpoint_time":"2024-05-06 13:32:44.286"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } table common_1.recover_and_insert not exists for 2-th check, retry later [Pipeline] // container =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:32:45 CST 2024] <<<<<< run test case storage_csv_update success! >>>>>> table test.finish_mark not exists for 20-th check, retry later [Pipeline] } [Pipeline] // withEnv [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318f7090000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-63kf5-dnt52-vw515, pid:6622, start at 2024-05-06 13:32:43.950988612 +0800 CST m=+27.036681839 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:43.958 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:43.940 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:43.940 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318f7090000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-63kf5-dnt52-vw515, pid:6622, start at 2024-05-06 13:32:43.950988612 +0800 CST m=+27.036681839 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:43.958 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:43.940 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:43.940 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318e4d04000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-63kf5-dnt52-vw515, pid:6713, start at 2024-05-06 13:32:25.293240278 +0800 CST m=+8.313603117 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:25.300 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:25.281 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:25.281 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] } [Pipeline] // withEnv [Pipeline] } table mark.finish_mark_3 not exists for 41-th check, retry later table test.table5 exists table test.table6 not exists for 1-th check, retry later [Pipeline] // stage table mark.finish_mark_1 not exists for 3-th check, retry later [Pipeline] } + set +x table processor_delay.t28 exists table processor_delay.t29 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 13:32:46 CST 2024] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.54695471.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table common_1.recover_and_insert exists table common_1.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 21-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:32:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0017946c0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570030580400134 MinTableBarrierTs:449570030580400136 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570030580400134, checkpointTs: 449570030580400134, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570030580400136, checkpointTs: 449570030580400136, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570030580400136, checkpointTs: 449570030580400136, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e5324d96-c835-465a-9217-682e07a8ee03 {"id":"e5324d96-c835-465a-9217-682e07a8ee03","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973564} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac43bb e5324d96-c835-465a-9217-682e07a8ee03 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570030580400134,"min-table-barrier-ts":449570030580400136,"admin-job-type":0} /tidb/cdc/default/default/task/position/e5324d96-c835-465a-9217-682e07a8ee03/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0017946c0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570030580400134 MinTableBarrierTs:449570030580400136 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570030580400134, checkpointTs: 449570030580400134, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570030580400136, checkpointTs: 449570030580400136, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570030580400136, checkpointTs: 449570030580400136, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e5324d96-c835-465a-9217-682e07a8ee03 {"id":"e5324d96-c835-465a-9217-682e07a8ee03","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973564} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac43bb e5324d96-c835-465a-9217-682e07a8ee03 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570030580400134,"min-table-barrier-ts":449570030580400136,"admin-job-type":0} /tidb/cdc/default/default/task/position/e5324d96-c835-465a-9217-682e07a8ee03/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0017946c0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570030580400134 MinTableBarrierTs:449570030580400136 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570030580400134, checkpointTs: 449570030580400134, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570030580400136, checkpointTs: 449570030580400136, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570030580400136, checkpointTs: 449570030580400136, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e5324d96-c835-465a-9217-682e07a8ee03 {"id":"e5324d96-c835-465a-9217-682e07a8ee03","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973564} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac43bb e5324d96-c835-465a-9217-682e07a8ee03 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570030580400134,"min-table-barrier-ts":449570030580400136,"admin-job-type":0} /tidb/cdc/default/default/task/position/e5324d96-c835-465a-9217-682e07a8ee03/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore table mark.finish_mark_3 not exists for 42-th check, retry later table test.table6 exists check diff failed 1-th time, retry later table mark.finish_mark_1 not exists for 4-th check, retry later Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t29 exists table processor_delay.t30 not exists for 1-th check, retry later [Mon May 6 13:32:47 CST 2024] <<<<<< START cdc server in multi_rocks case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.1916519167.out server --log-file /tmp/tidb_cdc_test/multi_rocks/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_rocks/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc already exists) "is-owner": true, run task successfully [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } table common_1.finish_mark not exists for 2-th check, retry later [Pipeline] // stage [Pipeline] } table mark.finish_mark_1 not exists for 5-th check, retry later table mark.finish_mark_3 not exists for 43-th check, retry later table test.finish_mark not exists for 22-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:32:49 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc53b832-d05b-47d1-91bc-294aed8c980b {"id":"cc53b832-d05b-47d1-91bc-294aed8c980b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973566} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63b5d0c6 cc53b832-d05b-47d1-91bc-294aed8c980b /tidb/cdc/default/default/upstream/7365755331741780569 {"id":7365755331741780569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc53b832-d05b-47d1-91bc-294aed8c980b {"id":"cc53b832-d05b-47d1-91bc-294aed8c980b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973566} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63b5d0c6 cc53b832-d05b-47d1-91bc-294aed8c980b /tidb/cdc/default/default/upstream/7365755331741780569 {"id":7365755331741780569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc53b832-d05b-47d1-91bc-294aed8c980b {"id":"cc53b832-d05b-47d1-91bc-294aed8c980b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973566} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63b5d0c6 cc53b832-d05b-47d1-91bc-294aed8c980b /tidb/cdc/default/default/upstream/7365755331741780569 {"id":7365755331741780569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: e03f5c6e-7cbc-4277-8ce8-22e23931d379 Info: {"upstream_id":7365755331741780569,"namespace":"default","id":"e03f5c6e-7cbc-4277-8ce8-22e23931d379","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-05-06T13:32:49.42039612+08:00","start_ts":449570030507524097,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570030507524097,"checkpoint_ts":449570030507524097,"checkpoint_time":"2024-05-06 13:32:46.084"} owner pid: 4988 owner id e5324d96-c835-465a-9217-682e07a8ee03 default owner key /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac43bb [Mon May 6 13:32:49 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.51285130.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully Resume changefeed with checkpointTs1 449570027232296961 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318fc4980016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd, pid:14293, start at 2024-05-06 13:32:49.3576516 +0800 CST m=+5.498540066 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:49.366 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:49.369 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:49.369 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t30 exists table processor_delay.t31 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table test.table3 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/kill_owner_with_ddl Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:32:50 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78df5da1-f487-4460-b52d-a6f169cedbdf {"id":"78df5da1-f487-4460-b52d-a6f169cedbdf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973567} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c635aa006 78df5da1-f487-4460-b52d-a6f169cedbdf /tidb/cdc/default/default/upstream/7365755232020950608 {"id":7365755232020950608,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78df5da1-f487-4460-b52d-a6f169cedbdf {"id":"78df5da1-f487-4460-b52d-a6f169cedbdf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973567} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c635aa006 78df5da1-f487-4460-b52d-a6f169cedbdf /tidb/cdc/default/default/upstream/7365755232020950608 {"id":7365755232020950608,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78df5da1-f487-4460-b52d-a6f169cedbdf {"id":"78df5da1-f487-4460-b52d-a6f169cedbdf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973567} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c635aa006 78df5da1-f487-4460-b52d-a6f169cedbdf /tidb/cdc/default/default/upstream/7365755232020950608 {"id":7365755232020950608,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.cli.19450.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 63b9bfbc-07e3-437c-9115-834d338bc969 Info: {"upstream_id":7365755232020950608,"namespace":"default","id":"63b9bfbc-07e3-437c-9115-834d338bc969","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:32:50.845959911+08:00","start_ts":449570031728066563,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570031728066563,"checkpoint_ts":449570031728066563,"checkpoint_time":"2024-05-06 13:32:50.740"} PASS table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... table common_1.finish_mark not exists for 3-th check, retry later table mark.finish_mark_1 not exists for 6-th check, retry later table mark.finish_mark_3 not exists for 44-th check, retry later table test.finish_mark not exists for 23-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318fc4980016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd, pid:14293, start at 2024-05-06 13:32:49.3576516 +0800 CST m=+5.498540066 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:49.366 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:49.369 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:49.369 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318fc1600013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd, pid:14374, start at 2024-05-06 13:32:49.132665466 +0800 CST m=+5.201470699 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:49.140 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:49.112 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:49.112 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t31 exists table processor_delay.t32 not exists for 1-th check, retry later table test.table3 not exists for 2-th check, retry later + set +x ***************** properties ***************** "insertproportion"="0" "mysql.user"="root" "recordcount"="1000" "readproportion"="0" "readallfields"="true" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "dotransactions"="false" "scanproportion"="0" "mysql.db"="multi_rocks" "threadcount"="2" "table"="a1" "workload"="core" "updateproportion"="0" "operationcount"="0" "mysql.port"="4000" ********************************************** table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:32:52 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035421b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570032153264138 MinTableBarrierTs:449570032153264142 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41692369-4719-4c4c-9ffe-9cf830dea5b5 {"id":"41692369-4719-4c4c-9ffe-9cf830dea5b5","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973570} /tidb/cdc/default/__cdc_meta__/capture/e5324d96-c835-465a-9217-682e07a8ee03 {"id":"e5324d96-c835-465a-9217-682e07a8ee03","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973564} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac43bb e5324d96-c835-465a-9217-682e07a8ee03 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac440e 41692369-4719-4c4c-9ffe-9cf830dea5b5 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570032153264138,"min-table-barrier-ts":449570032153264142,"admin-job-type":0} /tidb/cdc/default/default/task/position/41692369-4719-4c4c-9ffe-9cf830dea5b5/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e5324d96-c835-465a-9217-682e07a8ee03/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035421b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570032153264138 MinTableBarrierTs:449570032153264142 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41692369-4719-4c4c-9ffe-9cf830dea5b5 {"id":"41692369-4719-4c4c-9ffe-9cf830dea5b5","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973570} /tidb/cdc/default/__cdc_meta__/capture/e5324d96-c835-465a-9217-682e07a8ee03 {"id":"e5324d96-c835-465a-9217-682e07a8ee03","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973564} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac43bb e5324d96-c835-465a-9217-682e07a8ee03 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac440e 41692369-4719-4c4c-9ffe-9cf830dea5b5 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570032153264138,"min-table-barrier-ts":449570032153264142,"admin-job-type":0} /tidb/cdc/default/default/task/position/41692369-4719-4c4c-9ffe-9cf830dea5b5/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkp+ grep -q 'failed to get info:' oint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e5324d96-c835-465a-9217-682e07a8ee03/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035421b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570032153264138 MinTableBarrierTs:449570032153264142 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41692369-4719-4c4c-9ffe-9cf830dea5b5 {"id":"41692369-4719-4c4c-9ffe-9cf830dea5b5","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973570} /tidb/cdc/default/__cdc_meta__/capture/e5324d96-c835-465a-9217-682e07a8ee03 {"id":"e5324d96-c835-465a-9217-682e07a8ee03","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973564} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac43bb e5324d96-c835-465a-9217-682e07a8ee03 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac440e 41692369-4719-4c4c-9ffe-9cf830dea5b5 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570032153264138,"min-table-barrier-ts":449570032153264142,"admin-job-type":0} /tidb/cdc/default/default/task/position/41692369-4719-4c4c-9ffe-9cf830dea5b5/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e5324d96-c835-465a-9217-682e07a8ee03/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "e5324d96-c835-465a-9217-682e07a8ee03 default" | grep id Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc already exists) table common_1.finish_mark not exists for 4-th check, retry later Run finished, takes 588.172476ms INSERT - Takes(s): 0.6, Count: 1000, OPS: 1735.8, Avg(us): 1144, Min(us): 781, Max(us): 11993, 95th(us): 2000, 99th(us): 3000 ***************** properties ***************** "requestdistribution"="uniform" "table"="a2" "scanproportion"="0" "mysql.db"="multi_rocks" "readproportion"="0" "insertproportion"="0" "workload"="core" "mysql.host"="127.0.0.1" "recordcount"="1000" "operationcount"="0" "threadcount"="2" "updateproportion"="0" "mysql.port"="4000" "mysql.user"="root" "readallfields"="true" "dotransactions"="false" ********************************************** [Pipeline] // cache table mark.finish_mark_1 not exists for 7-th check, retry later [Pipeline] } table mark.finish_mark_3 not exists for 45-th check, retry later table test.finish_mark not exists for 24-th check, retry later [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } Run finished, takes 616.807189ms INSERT - Takes(s): 0.6, Count: 1000, OPS: 1667.8, Avg(us): 1202, Min(us): 840, Max(us): 17135, 95th(us): 2000, 99th(us): 3000 [Pipeline] // stage [Pipeline] } [Pipeline] // container VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318ff8340014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd, pid:30378, start at 2024-05-06 13:32:52.661264431 +0800 CST m=+5.195030392 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:52.667 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:52.671 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:52.671 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318ff8340014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd, pid:30378, start at 2024-05-06 13:32:52.661264431 +0800 CST m=+5.195030392 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:52.667 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:52.671 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:52.671 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d318ff8d40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd, pid:30466, start at 2024-05-06 13:32:52.709683633 +0800 CST m=+5.168316039 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:34:52.719 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:32:52.711 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:22:52.711 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/autorandom/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/autorandom/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] } ***************** properties ***************** "readallfields"="true" "mysql.user"="root" "updateproportion"="0" "mysql.port"="4000" "mysql.db"="multi_rocks" "operationcount"="0" "requestdistribution"="uniform" "recordcount"="1000" "table"="a3" "workload"="core" "scanproportion"="0" "insertproportion"="0" "threadcount"="2" "dotransactions"="false" "readproportion"="0" "mysql.host"="127.0.0.1" ********************************************** table processor_delay.t32 exists table processor_delay.t33 not exists for 1-th check, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } "id": "41692369-4719-4c4c-9ffe-9cf830dea5b5", run task successfully [Pipeline] // withEnv [Pipeline] } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.15668.out cli tso query --pd=http://127.0.0.1:2379 Run finished, takes 582.391458ms INSERT - Takes(s): 0.6, Count: 1000, OPS: 1766.7, Avg(us): 1133, Min(us): 812, Max(us): 16258, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "mysql.db"="multi_rocks" "operationcount"="0" "updateproportion"="0" "threadcount"="2" "table"="a4" "workload"="core" "requestdistribution"="uniform" "recordcount"="1000" "mysql.host"="127.0.0.1" "readproportion"="0" "mysql.port"="4000" "readallfields"="true" "mysql.user"="root" "insertproportion"="0" "dotransactions"="false" "scanproportion"="0" ********************************************** table ddl_only_block_related_table.finish_mark exists [Pipeline] // stage [Pipeline] } table test.table3 not exists for 3-th check, retry later table force_replicate_table.t0 exists table force_replicate_table.t1 exists table force_replicate_table.t2 not exists for 1-th check, retry later table common_1.finish_mark not exists for 5-th check, retry later Run finished, takes 537.169521ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1922.3, Avg(us): 1043, Min(us): 754, Max(us): 16907, 95th(us): 2000, 99th(us): 2000 wait process 7904 exit for 1-th time... table mark.finish_mark_3 not exists for 46-th check, retry later table test.finish_mark not exists for 25-th check, retry later ***************** properties ***************** "requestdistribution"="uniform" "updateproportion"="0" "scanproportion"="0" "readproportion"="0" "mysql.db"="multi_rocks" "insertproportion"="0" "threadcount"="2" "readallfields"="true" "dotransactions"="false" "recordcount"="1000" "operationcount"="0" "mysql.port"="4000" "table"="a5" "mysql.host"="127.0.0.1" "workload"="core" "mysql.user"="root" ********************************************** wait process 7904 exit for 2-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_1 not exists for 8-th check, retry later capture_id: 41692369-4719-4c4c-9ffe-9cf830dea5b5 1 ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '41692369-4719-4c4c-9ffe-9cf830dea5b5' 41692369-4719-4c4c-9ffe-9cf830dea5b5 run task successfully curl -X GET http://127.0.0.1:8300/status | grep '"is_owner": false' % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 21959 0 --:--:-- --:--:-- --:--:-- 22750 "is_owner": false run task successfully [Mon May 6 13:32:55 CST 2024] <<<<<< START cdc server in autorandom case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.autorandom.3178631788.out server --log-file /tmp/tidb_cdc_test/autorandom/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/autorandom/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449570032718446593 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570032718446593 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:32:55 CST 2024] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.1570915711.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_delay.t33 exists table processor_delay.t34 not exists for 1-th check, retry later /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (7904) - No such process wait process 7904 exit for 3-th time... process 7904 already exit [Mon May 6 13:32:55 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.80398041.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_nfs/sync_diff/output" source-instances = ["mysql1"] target-instance = "tidb0" target-check-tables = ["consistent_replicate_nfs.usertable*"] [data-sources] [data-sources.mysql1] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "449570028942262407" [data-sources.tidb0] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/05/06 13:32:54.134 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 13:32:54.135 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2024/05/06 13:32:54.135 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=:6060] [2024/05/06 13:32:54.135 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 13:32:54.135 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449570027054825582] [resolvedTs=449570028942262407] [2024/05/06 13:32:54.135 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=276.573ยตs] [2024/05/06 13:32:54.137 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 13:32:54.137 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 13:32:54.137 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2024/05/06 13:32:54.141 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CALLOW_INVALID_DATES%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 13:32:54.144 +08:00] [INFO] [mysql.go:197] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 13:32:54.146 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:32:54.146 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:32:54.146 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:32:54.146 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:32:54.146 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:32:54.146 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:32:54.146 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:32:54.146 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:32:54.146 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:32:54.146 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:32:54.146 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:32:54.146 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:32:54.146 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:32:54.146 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:32:54.146 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:32:54.146 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:32:54.146 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 13:32:54.146 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 13:32:54.146 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 13:32:54.146 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 13:32:54.146 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 13:32:54.146 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 13:32:54.146 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 13:32:54.146 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 13:32:54.146 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 13:32:54.146 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 13:32:54.146 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 13:32:54.146 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 13:32:54.146 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 13:32:54.146 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 13:32:54.146 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 13:32:54.146 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 13:32:54.146 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 13:32:54.146 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 13:32:54.146 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2024/05/06 13:32:54.150 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CALLOW_INVALID_DATES%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 13:32:54.152 +08:00] [INFO] [mysql_ddl_sink.go:107] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/06 13:32:54.175 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=40.189441ms] [2024/05/06 13:32:54.917 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}] [checkpointTs=449570028942262407] [2024/05/06 13:32:54.917 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}] [checkpointTs=449570028942262407] [2024/05/06 13:32:55.017 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=8953] [appliedDDLCount=0] [currentCheckpoint=449570028942262407] [2024/05/06 13:32:55.018 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 13:32:55.018 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 13:32:55.018 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/06 13:32:55.018 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 13:32:55.018 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 13:32:55.018 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/06 13:32:55.018 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 13:32:55.018 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 13:32:55.018 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 13:32:55.018 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 13:32:55.018 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/06 13:32:55.018 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/06 13:32:55.018 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/06 13:32:55.018 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 13:32:55.018 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/06 13:32:55.018 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] Apply redo log successfully check diff successfully [Mon May 6 13:32:55 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> count(*) 48027 Run finished, takes 875.27535ms INSERT - Takes(s): 0.8, Count: 999, OPS: 1285.4, Avg(us): 1619, Min(us): 795, Max(us): 97961, 95th(us): 2000, 99th(us): 3000 table test.table3 not exists for 4-th check, retry later table force_replicate_table.t2 exists table force_replicate_table.t3 not exists for 1-th check, retry later table multi_rocks.finish_mark not exists for 1-th check, retry later table common_1.finish_mark exists table test.finish_mark not exists for 26-th check, retry later check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_1 not exists for 9-th check, retry later table mark.finish_mark_3 not exists for 47-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table processor_delay.t34 exists table processor_delay.t35 not exists for 1-th check, retry later table test.table3 exists table test.table4 not exists for 1-th check, retry later wait process cdc.test exit for 3-th time... table force_replicate_table.t3 exists table force_replicate_table.t4 not exists for 1-th check, retry later table multi_rocks.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 27-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:32:58 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/af2751e1-bc09-413f-bcac-27d92d93a6b6 {"id":"af2751e1-bc09-413f-bcac-27d92d93a6b6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973576} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63d7f1c6 af2751e1-bc09-413f-bcac-27d92d93a6b6 /tidb/cdc/default/default/upstream/7365755373826585969 {"id":7365755373826585969,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/af2751e1-bc09-413f-bcac-27d92d93a6b6 {"id":"af2751e1-bc09-413f-bcac-27d92d93a6b6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973576} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63d7f1c6 af2751e1-bc09-413f-bcac-27d92d93a6b6 /tidb/cdc/default/default/upstream/7365755373826585969 {"id":7365755373826585969,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/af2751e1-bc09-413f-bcac-27d92d93a6b6 {"id":"af2751e1-bc09-413f-bcac-27d92d93a6b6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973576} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63d7f1c6 af2751e1-bc09-413f-bcac-27d92d93a6b6 /tidb/cdc/default/default/upstream/7365755373826585969 {"id":7365755373826585969,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 89eed05a-4177-4386-8c90-d46b5c8031f2 Info: {"upstream_id":7365755373826585969,"namespace":"default","id":"89eed05a-4177-4386-8c90-d46b5c8031f2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:32:59.087218494+08:00","start_ts":449570033886035970,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570033886035970,"checkpoint_ts":449570033886035970,"checkpoint_time":"2024-05-06 13:32:58.972"} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:32:59 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1931cbe-3a53-4b08-877c-1a0f1a52eec7 {"id":"f1931cbe-3a53-4b08-877c-1a0f1a52eec7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973576} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63c403d3 f1931cbe-3a53-4b08-877c-1a0f1a52eec7 /tidb/cdc/default/default/upstream/7365755355441278641 {"id":7365755355441278641,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1931cbe-3a53-4b08-877c-1a0f1a52eec7 {"id":"f1931cbe-3a53-4b08-877c-1a0f1a52eec7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973576} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63c403d3 f1931cbe-3a53-4b08-877c-1a0f1a52eec7 /tidb/cdc/default/default/upstream/7365755355441278641 {"id":7365755355441278641,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1931cbe-3a53-4b08-877c-1a0f1a52eec7 {"id":"f1931cbe-3a53-4b08-877c-1a0f1a52eec7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973576} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63c403d3 f1931cbe-3a53-4b08-877c-1a0f1a52eec7 /tidb/cdc/default/default/upstream/7365755355441278641 {"id":7365755355441278641,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.15771.out cli changefeed create --start-ts=449570032718446593 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:32:59 CST 2024] <<<<<< run test case common_1 success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:32:59 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365755301910702846 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 13:32:44.395107764 +0800 CST StartTs:449570030036189189 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0024d0240 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570030062403586} {CheckpointTs:449570033797955588 MinTableBarrierTs:449570033797955588 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570033797955588, checkpointTs: 449570033797955588, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570033797955588, checkpointTs: 449570033797955588, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449570033797955588, checkpointTs: 449570033797955588, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449570033797955588, checkpointTs: 449570033797955588, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/93e95531-9e66-4983-b387-72bafeb874ad {"id":"93e95531-9e66-4983-b387-72bafeb874ad","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973576} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c639021a9 93e95531-9e66-4983-b387-72bafeb874ad /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365755301910702846,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T13:32:44.395107764+08:00","start-ts":449570030036189189,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570030062403586} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449570033797955588,"min-table-barrier-ts":449570033797955588,"admin-job-type":0} /tidb/cdc/default/default/task/position/93e95531-9e66-4983-b387-72bafeb874ad/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755301910702846 {"id":7365755301910702846,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365755301910702846 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 13:32:44.395107764 +0800 CST StartTs:449570030036189189 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0024d0240 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570030062403586} {CheckpointTs:449570033797955588 MinTableBarrierTs:449570033797955588 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570033797955588, checkpointTs: 449570033797955588, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570033797955588, checkpointTs: 449570033797955588, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449570033797955588, checkpointTs: 449570033797955588, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449570033797955588, checkpointTs: 449570033797955588, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/93e95531-9e66-4983-b387-72bafeb874ad {"id":"93e95531-9e66-4983-b387-72bafeb874ad","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973576} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c639021a9 93e95531-9e66-4983-b387-72bafeb874ad /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365755301910702846,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T13:32:44.395107764+08:00","start-ts":449570030036189189,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570030062403586} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449570033797955588,"min-table-barrier-ts":449570033797955588,"admin-job-type":0} /tidb/cdc/default/default/task/position/93e95531-9e66-4983-b387-72bafeb874ad/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755301910702846 {"id":7365755301910702846,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365755301910702846 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 13:32:44.395107764 +0800 CST StartTs:449570030036189189 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0024d0240 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570030062403586} {CheckpointTs:449570033797955588 MinTableBarrierTs:449570033797955588 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570033797955588, checkpointTs: 449570033797955588, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570033797955588, checkpointTs: 449570033797955588, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449570033797955588, checkpointTs: 449570033797955588, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449570033797955588, checkpointTs: 449570033797955588, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/93e95531-9e66-4983-b387-72bafeb874ad {"id":"93e95531-9e66-4983-b387-72bafeb874ad","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973576} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c639021a9 93e95531-9e66-4983-b387-72bafeb874ad /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365755301910702846,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T13:32:44.395107764+08:00","start-ts":449570030036189189,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570030062403586} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449570033797955588,"min-table-barrier-ts":449570033797955588,"admin-job-type":0} /tidb/cdc/default/default/task/position/93e95531-9e66-4983-b387-72bafeb874ad/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755301910702846 {"id":7365755301910702846,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table autorandom_test.table_a not exists for 1-th check, retry later Create changefeed successfully! ID: d17cfe5a-15fd-4b61-8ee3-ef3c634c849d Info: {"upstream_id":7365755355441278641,"namespace":"default","id":"d17cfe5a-15fd-4b61-8ee3-ef3c634c849d","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:32:59.494519812+08:00","start_ts":449570032718446593,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570032718446593,"checkpoint_ts":449570032718446593,"checkpoint_time":"2024-05-06 13:32:54.518"} PASS table mark.finish_mark_1 not exists for 10-th check, retry later table mark.finish_mark_3 not exists for 48-th check, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later table processor_delay.t35 exists table processor_delay.t36 not exists for 1-th check, retry later table test.table4 exists check diff successfully table force_replicate_table.t4 exists table force_replicate_table.t5 not exists for 1-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later wait process cdc.test exit for 1-th time... + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31906db40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-0cr08-7wrc3-1z3gv, pid:6920, start at 2024-05-06 13:33:00.157001343 +0800 CST m=+5.074538721 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:35:00.162 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:33:00.141 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:23:00.141 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31906db40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-0cr08-7wrc3-1z3gv, pid:6920, start at 2024-05-06 13:33:00.157001343 +0800 CST m=+5.074538721 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:35:00.162 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:33:00.141 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:23:00.141 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31906f200015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-0cr08-7wrc3-1z3gv, pid:6999, start at 2024-05-06 13:33:00.274089889 +0800 CST m=+5.128399189 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:35:00.280 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:33:00.283 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:23:00.283 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table autorandom_test.table_a exists table savepoint.finish_mark not exists for 1-th check, retry later table mark.finish_mark_1 not exists for 11-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table mark.finish_mark_3 not exists for 49-th check, retry later table test.finish_mark not exists for 28-th check, retry later check diff failed 1-th time, retry later table multi_rocks.finish_mark not exists for 3-th check, retry later table processor_delay.t36 exists table processor_delay.t37 not exists for 1-th check, retry later check diff failed 2-th time, retry later [2024/05/06 13:32:58.482 +08:00] [INFO] [case.go:115] ["sync updatePKUK take: 11.465689595s"] table force_replicate_table.t5 exists table force_replicate_table.t6 not exists for 1-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2707 0 --:--:-- --:--:-- --:--:-- 2728 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 13:32:49.245","puller_resolved_ts":"2024-05-06 13:32:42.196","last_synced_ts":"2024-05-06 13:30:33.846","now_ts":"2024-05-06 13:32:51.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '13:32:49.245","puller_resolved_ts":"2024-05-06' '13:32:42.196","last_synced_ts":"2024-05-06' '13:30:33.846","now_ts":"2024-05-06' '13:32:51.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 9212 8.7 0.0 14030636 143548 ? Sl 13:30 0:13 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 9274 5.9 0.0 13751268 137128 ? Sl 13:30 0:09 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1714973576.5284169,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00075ec40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714973576.5284967,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714973576.5418184,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00399a000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714973576.5419087,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714973577.2579632,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002360380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714973577.258051,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T13:33:01.098096+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001387500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:33:01.103427+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012ba000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:33:01.188948+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00132f6c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} wait process cdc.test exit for 2-th time... table multi_rocks.finish_mark not exists for 4-th check, retry later cdc.test: no process found check diff successfully wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:33:03 CST 2024] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=1*return(true)' + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.88238825.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 50-th check, retry later table test.finish_mark not exists for 29-th check, retry later table savepoint.finish_mark not exists for 2-th check, retry later table mark.finish_mark_1 not exists for 12-th check, retry later [Mon May 6 13:33:03 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.82598261.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... table processor_delay.t37 exists table processor_delay.t38 not exists for 1-th check, retry later check diff successfully check_ts_not_forward ddl-only-block-related-table table force_replicate_table.t6 not exists for 2-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_1 not exists for 13-th check, retry later table mark.finish_mark_3 not exists for 51-th check, retry later table test.finish_mark not exists for 30-th check, retry later table savepoint.finish_mark exists check diff successfully wait process cdc.test exit for 2-th time... table multi_rocks.finish_mark not exists for 5-th check, retry later table processor_delay.t38 exists table processor_delay.t39 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:33:06 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a21f75fa-572c-4437-826b-805e0799190f {"id":"a21f75fa-572c-4437-826b-805e0799190f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973583} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63efd9c6 a21f75fa-572c-4437-826b-805e0799190f /tidb/cdc/default/default/upstream/7365755405094065760 {"id":7365755405094065760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a21f75fa-572c-4437-826b-805e0799190f {"id":"a21f75fa-572c-4437-826b-805e0799190f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973583} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63efd9c6 a21f75fa-572c-4437-826b-805e0799190f /tidb/cdc/default/default/upstream/7365755405094065760 {"id":7365755405094065760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a21f75fa-572c-4437-826b-805e0799190f {"id":"a21f75fa-572c-4437-826b-805e0799190f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973583} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63efd9c6 a21f75fa-572c-4437-826b-805e0799190f /tidb/cdc/default/default/upstream/7365755405094065760 {"id":7365755405094065760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:33:06 CST 2024] <<<<<< run test case autorandom success! >>>>>> wait process cdc.test exit for 2-th time... Create changefeed successfully! ID: 2eafe328-2689-494f-a8d8-cf680204606a Info: {"upstream_id":7365755405094065760,"namespace":"default","id":"2eafe328-2689-494f-a8d8-cf680204606a","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T13:33:06.670803611+08:00","start_ts":449570035870203909,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570035870203909,"checkpoint_ts":449570035870203909,"checkpoint_time":"2024-05-06 13:33:06.541"} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:33:06 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e5a1951b-88f1-4b2e-ba78-a608919787b2 {"id":"e5a1951b-88f1-4b2e-ba78-a608919787b2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973583} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4c636e2c80 e5a1951b-88f1-4b2e-ba78-a608919787b2 /tidb/cdc/default/default/changefeed/info/46fd563b-d75f-4ed4-86ef-a644c41e39ad {"upstream-id":7365755260047450314,"namespace":"default","changefeed-id":"46fd563b-d75f-4ed4-86ef-a644c41e39ad","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:32:33.476776448+08:00","start-ts":449570027232296961,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"failed","error":{"time":"2024-05-06T13:33:03.75808034+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570035136200706} /tidb/cdc/default/default/changefeed/status/46fd563b-d75f-4ed4-86ef-a644c41e39ad {"checkpoint-ts":449570029093781527,"min-table-barrier-ts":449570029119995933,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365755260047450314 {"id":7365755260047450314,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e5a1951b-88f1-4b2e-ba78-a608919787b2 {"id":"e5a1951b-88f1-4b2e-ba78-a608919787b2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973583} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4c636e2c80 e5a1951b-88f1-4b2e-ba78-a608919787b2 /tidb/cdc/default/default/changefeed/info/46fd563b-d75f-4ed4-86ef-a644c41e39ad {"upstream-id":7365755260047450314,"namespace":"default","changefeed-id":"46fd563b-d75f-4ed4-86ef-a644c41e39ad","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:32:33.476776448+08:00","start-ts":449570027232296961,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"failed","error":{"time":"2024-05-06T13:33:03.75808034+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570035136200706} /tidb/cdc/default/default/changefeed/status/46fd563b-d75f-4ed4-86ef-a644c41e39ad {"checkpoint-ts":449570029093781527,"min-table-barrier-ts":449570029119995933,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365755260047450314 {"id":7365755260047450314,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e5a1951b-88f1-4b2e-ba78-a608919787b2 {"id":"e5a1951b-88f1-4b2e-ba78-a608919787b2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973583} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4c636e2c80 e5a1951b-88f1-4b2e-ba78-a608919787b2 /tidb/cdc/default/default/changefeed/info/46fd563b-d75f-4ed4-86ef-a644c41e39ad {"upstream-id":7365755260047450314,"namespace":"default","changefeed-id":"46fd563b-d75f-4ed4-86ef-a644c41e39ad","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:32:33.476776448+08:00","start-ts":449570027232296961,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"failed","error":{"time":"2024-05-06T13:33:03.75808034+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570035136200706} /tidb/cdc/default/default/changefeed/status/46fd563b-d75f-4ed4-86ef-a644c41e39ad {"checkpoint-ts":449570029093781527,"min-table-barrier-ts":449570029119995933,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365755260047450314 {"id":7365755260047450314,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 46fd563b-d75f-4ed4-86ef-a644c41e39ad failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=46fd563b-d75f-4ed4-86ef-a644c41e39ad + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 46fd563b-d75f-4ed4-86ef-a644c41e39ad -s + info='{ "upstream_id": 7365755260047450314, "namespace": "default", "id": "46fd563b-d75f-4ed4-86ef-a644c41e39ad", "state": "failed", "checkpoint_tso": 449570029093781527, "checkpoint_time": "2024-05-06 13:32:40.691", "error": { "time": "2024-05-06T13:33:03.75808034+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7365755260047450314, "namespace": "default", "id": "46fd563b-d75f-4ed4-86ef-a644c41e39ad", "state": "failed", "checkpoint_tso": 449570029093781527, "checkpoint_time": "2024-05-06 13:32:40.691", "error": { "time": "2024-05-06T13:33:03.75808034+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7365755260047450314, "namespace": "default", "id": "46fd563b-d75f-4ed4-86ef-a644c41e39ad", "state": "failed", "checkpoint_tso": 449570029093781527, "checkpoint_time": "2024-05-06 13:32:40.691", "error": { "time": "2024-05-06T13:33:03.75808034+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365755260047450314, '"namespace":' '"default",' '"id":' '"46fd563b-d75f-4ed4-86ef-a644c41e39ad",' '"state":' '"failed",' '"checkpoint_tso":' 449570029093781527, '"checkpoint_time":' '"2024-05-06' '13:32:40.691",' '"error":' '{' '"time":' '"2024-05-06T13:33:03.75808034+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7365755260047450314, '"namespace":' '"default",' '"id":' '"46fd563b-d75f-4ed4-86ef-a644c41e39ad",' '"state":' '"failed",' '"checkpoint_tso":' 449570029093781527, '"checkpoint_time":' '"2024-05-06' '13:32:40.691",' '"error":' '{' '"time":' '"2024-05-06T13:33:03.75808034+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully table force_replicate_table.t6 exists check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table kill_owner_with_ddl.t1 not exists for 1-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:33:06 CST 2024] <<<<<< run test case savepoint success! >>>>>> run task successfully check_data_subset force_replicate_table.t1 127.0.0.1 4000 127.0.0.1 3306 table mark.finish_mark_3 not exists for 52-th check, retry later table test.finish_mark not exists for 31-th check, retry later empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check_changefeed_state http://127.0.0.1:2379 46fd563b-d75f-4ed4-86ef-a644c41e39ad normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=46fd563b-d75f-4ed4-86ef-a644c41e39ad + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 46fd563b-d75f-4ed4-86ef-a644c41e39ad -s + info='{ "upstream_id": 7365755260047450314, "namespace": "default", "id": "46fd563b-d75f-4ed4-86ef-a644c41e39ad", "state": "normal", "checkpoint_tso": 449570035961954307, "checkpoint_time": "2024-05-06 13:33:06.891", "error": null }' + echo '{ "upstream_id": 7365755260047450314, "namespace": "default", "id": "46fd563b-d75f-4ed4-86ef-a644c41e39ad", "state": "normal", "checkpoint_tso": 449570035961954307, "checkpoint_time": "2024-05-06 13:33:06.891", "error": null }' { "upstream_id": 7365755260047450314, "namespace": "default", "id": "46fd563b-d75f-4ed4-86ef-a644c41e39ad", "state": "normal", "checkpoint_tso": 449570035961954307, "checkpoint_time": "2024-05-06 13:33:06.891", "error": null } ++ echo '{' '"upstream_id":' 7365755260047450314, '"namespace":' '"default",' '"id":' '"46fd563b-d75f-4ed4-86ef-a644c41e39ad",' '"state":' '"normal",' '"checkpoint_tso":' 449570035961954307, '"checkpoint_time":' '"2024-05-06' '13:33:06.891",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365755260047450314, '"namespace":' '"default",' '"id":' '"46fd563b-d75f-4ed4-86ef-a644c41e39ad",' '"state":' '"normal",' '"checkpoint_tso":' 449570035961954307, '"checkpoint_time":' '"2024-05-06' '13:33:06.891",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table test1.table1 not exists for 1-th check, retry later {"level":"warn","ts":"2024-05-06T13:33:07.099256+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001387500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:33:07.104595+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012ba000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:33:07.190498+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00132f6c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table mark.finish_mark_1 not exists for 14-th check, retry later table multi_rocks.finish_mark not exists for 6-th check, retry later run task successfully check_data_subset force_replicate_table.t2 127.0.0.1 4000 127.0.0.1 3306 table processor_delay.t39 exists table processor_delay.t40 not exists for 1-th check, retry later run task successfully check_data_subset force_replicate_table.t3 127.0.0.1 4000 127.0.0.1 3306 empty select id, val from test.availability1 where id=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table test.finish_mark not exists for 32-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table kill_owner_with_ddl.t1 exists wait process 8264 exit for 1-th time... [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } table mark.finish_mark_1 not exists for 15-th check, retry later table multi_rocks.finish_mark exists check diff successfully table mark.finish_mark_3 not exists for 53-th check, retry later [Pipeline] // stage [Pipeline] } wait process 8264 exit for 2-th time... run task successfully check_data_subset force_replicate_table.t4 127.0.0.1 4000 127.0.0.1 3306 [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } wait process cdc.test exit for 1-th time... table test1.table1 not exists for 2-th check, retry later [Pipeline] // node table processor_delay.t40 exists table processor_delay.t41 not exists for 1-th check, retry later [Pipeline] } [Pipeline] // podTemplate [Pipeline] } /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8264) - No such process wait process 8264 exit for 3-th time... process 8264 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 13:33:10 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.84438445.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 run task successfully check_data_subset force_replicate_table.t5 127.0.0.1 4000 127.0.0.1 3306 [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } wait process cdc.test exit for 2-th time... table test.finish_mark not exists for 33-th check, retry later run task successfully check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 id=19,a=NULL doesn't exist in downstream table force_replicate_table.t6 run task failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:33:11 CST 2024] <<<<<< run test case multi_rocks success! >>>>>> table mark.finish_mark_1 exists table mark.finish_mark_2 not exists for 1-th check, retry later nonempty select id, val from test.availability2 where id=1 and val=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_3 not exists for 54-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed table test1.table1 not exists for 3-th check, retry later table processor_delay.t41 exists table processor_delay.t42 not exists for 1-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0{"level":"warn","ts":"2024-05-06T13:33:13.100609+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001387500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:33:13.105301+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012ba000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:33:13.191744+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00132f6c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table mark.finish_mark_2 not exists for 2-th check, retry later nonempty select id, val from test.availability2 where id=1 and val=22 run task successfully empty select id, val from test.availability2 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_3 not exists for 55-th check, retry later table test.finish_mark not exists for 34-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:33:13 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/2eafe328-2689-494f-a8d8-cf680204606a {UpstreamID:7365755405094065760 Namespace:default ID:2eafe328-2689-494f-a8d8-cf680204606a SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:33:06.670803611 +0800 CST StartTs:449570035870203909 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f92090 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570035896418311} {CheckpointTs:449570037600354317 MinTableBarrierTs:449570037600354317 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570037600354317, checkpointTs: 449570037600354317, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7af80e78-0c36-4a8a-9c4b-9bf515c3bce8 {"id":"7af80e78-0c36-4a8a-9c4b-9bf515c3bce8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973590} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63efda46 7af80e78-0c36-4a8a-9c4b-9bf515c3bce8 /tidb/cdc/default/default/changefeed/info/2eafe328-2689-494f-a8d8-cf680204606a {"upstream-id":7365755405094065760,"namespace":"default","changefeed-id":"2eafe328-2689-494f-a8d8-cf680204606a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:33:06.670803611+08:00","start-ts":449570035870203909,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570035896418311} /tidb/cdc/default/default/changefeed/status/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":449570037600354317,"min-table-barrier-ts":449570037600354317,"admin-job-type":0} /tidb/cdc/default/default/task/position/7af80e78-0c36-4a8a-9c4b-9bf515c3bce8/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755405094065760 {"id":7365755405094065760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2eafe328-2689-494f-a8d8-cf680204606a {UpstreamID:7365755405094065760 Namespace:default ID:2eafe328-2689-494f-a8d8-cf680204606a SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:33:06.670803611 +0800 CST StartTs:449570035870203909 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f92090 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570035896418311} {CheckpointTs:449570037600354317 MinTableBarrierTs:449570037600354317 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570037600354317, checkpointTs: 449570037600354317, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7af80e78-0c36-4a8a-9c4b-9bf515c3bce8 {"id":"7af80e78-0c36-4a8a-9c4b-9bf515c3bce8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973590} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63efda46 7af80e78-0c36-4a8a-9c4b-9bf515c3bce8 /tidb/cdc/default/default/changefeed/info/2eafe328-2689-494f-a8d8-cf680204606a {"upstream-id":7365755405094065760,"namespace":"default","changefeed-id":"2eafe328-2689-494f-a8d8-cf680204606a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:33:06.670803611+08:00","start-ts":449570035870203909,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570035896418311} /tidb/cdc/default/default/changefeed/status/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":449570037600354317,"min-table-barrier-ts":449570037600354317,"admin-job-type":0} /tidb/cdc/default/default/task/position/7af80e78-0c36-4a8a-9c4b-9bf515c3bce8/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755405094065760 {"id":7365755405094065760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2eafe328-2689-494f-a8d8-cf680204606a {UpstreamID:7365755405094065760 Namespace:default ID:2eafe328-2689-494f-a8d8-cf680204606a SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:33:06.670803611 +0800 CST StartTs:449570035870203909 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f92090 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570035896418311} {CheckpointTs:449570037600354317 MinTableBarrierTs:449570037600354317 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570037600354317, checkpointTs: 449570037600354317, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7af80e78-0c36-4a8a-9c4b-9bf515c3bce8 {"id":"7af80e78-0c36-4a8a-9c4b-9bf515c3bce8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973590} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63efda46 7af80e78-0c36-4a8a-9c4b-9bf515c3bce8 /tidb/cdc/default/default/changefeed/info/2eafe328-2689-494f-a8d8-cf680204606a {"upstream-id":7365755405094065760,"namespace":"default","changefeed-id":"2eafe328-2689-494f-a8d8-cf680204606a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:33:06.670803611+08:00","start-ts":449570035870203909,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570035896418311} /tidb/cdc/default/default/changefeed/status/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":449570037600354317,"min-table-barrier-ts":449570037600354317,"admin-job-type":0} /tidb/cdc/default/default/task/position/7af80e78-0c36-4a8a-9c4b-9bf515c3bce8/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755405094065760 {"id":7365755405094065760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully run task successfully table test1.table1 exists table test1.table2 not exists for 1-th check, retry later table processor_delay.t42 exists table processor_delay.t43 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... empty select id, val from test.availability2 where id=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_3 not exists for 56-th check, retry later table test.finish_mark not exists for 35-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:33:15 CST 2024] <<<<<< run test case force_replicate_table success! >>>>>> table mark.finish_mark_2 not exists for 3-th check, retry later table test1.table2 exists check diff successfully table processor_delay.t43 exists table processor_delay.t44 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/generate_column/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 57-th check, retry later table test.finish_mark not exists for 36-th check, retry later table mark.finish_mark_2 not exists for 4-th check, retry later wait process cdc.test exit for 1-th time... nonempty select id, val from test.availability3 where id=1 and val=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... table processor_delay.t44 exists table processor_delay.t45 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:33:18 CST 2024] <<<<<< run test case changefeed_resume_with_checkpoint_ts success! >>>>>> 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0{"level":"warn","ts":"2024-05-06T13:33:19.101531+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001387500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:33:19.106473+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012ba000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:33:19.193329+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00132f6c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table test.finish_mark not exists for 37-th check, retry later table mark.finish_mark_2 not exists for 5-th check, retry later nonempty select id, val from test.availability3 where id=1 and val=22 run task successfully empty select id, val from test.availability3 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 58-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... wait process 8448 exit for 1-th time... wait process 8448 exit for 2-th time... table processor_delay.t45 exists table processor_delay.t46 not exists for 1-th check, retry later /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8448) - No such process wait process 8448 exit for 3-th time... process 8448 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 13:33:20 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.86488650.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 38-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0{"level":"warn","ts":"2024-05-06T13:33:21.088611+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001387500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-06T13:33:21.088675+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T13:33:21.092037+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012ba000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-06T13:33:21.09209+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T13:33:21.179829+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00132f6c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-06T13:33:21.179893+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} table mark.finish_mark_2 not exists for 6-th check, retry later empty select id, val from test.availability3 where id=1 run task successfully delete_owner_key pass VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3191b2940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz, pid:9703, start at 2024-05-06 13:33:20.960441582 +0800 CST m=+5.597093198 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:35:20.970 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:33:20.933 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:23:20.933 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 59-th check, retry later wait process cdc.test exit for 1-th time... table processor_delay.t46 exists table processor_delay.t47 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... table mark.finish_mark_2 not exists for 7-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit run test case test_kill_capture [Mon May 6 13:33:23 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.56035605.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3191b2940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz, pid:9703, start at 2024-05-06 13:33:20.960441582 +0800 CST m=+5.597093198 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:35:20.970 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:33:20.933 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:23:20.933 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3191b398000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz, pid:9778, start at 2024-05-06 13:33:21.006042095 +0800 CST m=+5.577033208 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:35:21.014 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:33:20.998 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:23:20.998 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 39-th check, retry later table mark.finish_mark_3 not exists for 60-th check, retry later [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:33:23 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/2eafe328-2689-494f-a8d8-cf680204606a {UpstreamID:7365755405094065760 Namespace:default ID:2eafe328-2689-494f-a8d8-cf680204606a SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:33:06.670803611 +0800 CST StartTs:449570035870203909 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001289170 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570035896418311} {CheckpointTs:449570037757640734 MinTableBarrierTs:449570037757640734 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570037757640734, checkpointTs: 449570037757640734, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92157539-ddb7-445c-98f7-cb35a226403c {"id":"92157539-ddb7-445c-98f7-cb35a226403c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973601} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63efdacf 92157539-ddb7-445c-98f7-cb35a226403c /tidb/cdc/default/default/changefeed/info/2eafe328-2689-494f-a8d8-cf680204606a {"upstream-id":7365755405094065760,"namespace":"default","changefeed-id":"2eafe328-2689-494f-a8d8-cf680204606a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:33:06.670803611+08:00","start-ts":449570035870203909,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570035896418311} /tidb/cdc/default/default/changefeed/status/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":449570037757640734,"min-table-barrier-ts":449570037757640734,"admin-job-type":0} /tidb/cdc/default/default/task/position/92157539-ddb7-445c-98f7-cb35a226403c/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755405094065760 {"id":7365755405094065760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2eafe328-2689-494f-a8d8-cf680204606a {UpstreamID:7365755405094065760 Namespace:default ID:2eafe328-2689-494f-a8d8-cf680204606a SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:33:06.670803611 +0800 CST StartTs:449570035870203909 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001289170 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570035896418311} {CheckpointTs:449570037757640734 MinTableBarrierTs:449570037757640734 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570037757640734, checkpointTs: 449570037757640734, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92157539-ddb7-445c-98f7-cb35a226403c {"id":"92157539-ddb7-445c-98f7-cb35a226403c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973601} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63efdacf 92157539-ddb7-445c-98f7-cb35a226403c /tidb/cdc/default/default/changefeed/info/2eafe328-2689-494f-a8d8-cf680204606a {"upstream-id":7365755405094065760,"namespace":"default","changefeed-id":"2eafe328-2689-494f-a8d8-cf680204606a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:33:06.670803611+08:00","start-ts":449570035870203909,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570035896418311} /tidb/cdc/default/default/changefeed/status/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":449570037757640734,"min-table-barrier-ts":449570037757640734,"admin-job-type":0} /tidb/cdc/default/default/task/position/92157539-ddb7-445c-98f7-cb35a226403c/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755405094065760 {"id":7365755405094065760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2eafe328-2689-494f-a8d8-cf680204606a {UpstreamID:7365755405094065760 Namespace:default ID:2eafe328-2689-494f-a8d8-cf680204606a SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:33:06.670803611 +0800 CST StartTs:449570035870203909 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001289170 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570035896418311} {CheckpointTs:449570037757640734 MinTableBarrierTs:449570037757640734 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570037757640734, checkpointTs: 449570037757640734, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92157539-ddb7-445c-98f7-cb35a226403c {"id":"92157539-ddb7-445c-98f7-cb35a226403c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973601} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63efdacf 92157539-ddb7-445c-98f7-cb35a226403c /tidb/cdc/default/default/changefeed/info/2eafe328-2689-494f-a8d8-cf680204606a {"upstream-id":7365755405094065760,"namespace":"default","changefeed-id":"2eafe328-2689-494f-a8d8-cf680204606a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:33:06.670803611+08:00","start-ts":449570035870203909,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570035896418311} /tidb/cdc/default/default/changefeed/status/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":449570037757640734,"min-table-barrier-ts":449570037757640734,"admin-job-type":0} /tidb/cdc/default/default/task/position/92157539-ddb7-445c-98f7-cb35a226403c/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755405094065760 {"id":7365755405094065760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully [Pipeline] // stage [Pipeline] } table processor_delay.t47 exists table processor_delay.t48 not exists for 1-th check, retry later [Pipeline] // container [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0{"level":"warn","ts":"2024-05-06T13:33:25.103375+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001387500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:33:25.107628+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012ba000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } {"level":"warn","ts":"2024-05-06T13:33:25.194049+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00132f6c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table test.finish_mark exists check diff successfully table mark.finish_mark_3 not exists for 61-th check, retry later table mark.finish_mark_2 exists table mark.finish_mark_3 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... [Mon May 6 13:33:25 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file.1114611148.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_delay.t48 exists table processor_delay.t49 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:33:26 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00167a750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570040816861194 MinTableBarrierTs:449570041079267336 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570041079267336, checkpointTs: 449570040816861197, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570041079267334, checkpointTs: 449570040816861197, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570041079267336, checkpointTs: 449570040816861197, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/34206108-c200-4916-897e-35edd3bdd293 {"id":"34206108-c200-4916-897e-35edd3bdd293","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973603} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4587 34206108-c200-4916-897e-35edd3bdd293 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570040816861197,"min-table-barrier-ts":449570041079267336,"admin-job-type":0} /tidb/cdc/default/default/task/position/34206108-c200-4916-897e-35edd3bdd293/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00167a750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570040816861194 MinTableBarrierTs:449570041079267336 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570041079267336, checkpointTs: 449570040816861197, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570041079267334, checkpointTs: 449570040816861197, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570041079267336, checkpointTs: 449570040816861197, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/34206108-c200-4916-897e-35edd3bdd293 {"id":"34206108-c200-4916-897e-35edd3bdd293","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973603} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4587 34206108-c200-4916-897e-35edd3bdd293 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570040816861197,"min-table-barrier-ts":449570041079267336,"admin-job-type":0} /tidb/cdc/default/default/task/position/34206108-c200-4916-897e-35edd3bdd293/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00167a750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570040816861194 MinTableBarrierTs:449570041079267336 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570041079267336, checkpointTs: 449570040816861197, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570041079267334, checkpointTs: 449570040816861197, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570041079267336, checkpointTs: 449570040816861197, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/34206108-c200-4916-897e-35edd3bdd293 {"id":"34206108-c200-4916-897e-35edd3bdd293","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973603} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4587 34206108-c200-4916-897e-35edd3bdd293 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570040816861197,"min-table-barrier-ts":449570041079267336,"admin-job-type":0} /tidb/cdc/default/default/task/position/34206108-c200-4916-897e-35edd3bdd293/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' wait process cdc.test exit for 3-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:33:27 CST 2024] <<<<<< run test case tidb_mysql_test success! >>>>>> table mark.finish_mark_3 not exists for 62-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc already exists) table mark.finish_mark_3 not exists for 2-th check, retry later [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // timeout "is-owner": true, run task successfully [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } table processor_delay.t49 exists table processor_delay.t50 not exists for 1-th check, retry later [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // withEnv [Pipeline] } [2024/05/06 13:33:26.102 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 5 another is 4"] [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:33:28 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49a694eb-cb0e-4bce-ab68-aa417b054c63 {"id":"49a694eb-cb0e-4bce-ab68-aa417b054c63","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973606} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c643f1cca 49a694eb-cb0e-4bce-ab68-aa417b054c63 /tidb/cdc/default/default/upstream/7365755495359373414 {"id":7365755495359373414,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49a694eb-cb0e-4bce-ab68-aa417b054c63 {"id":"49a694eb-cb0e-4bce-ab68-aa417b054c63","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973606} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c643f1cca 49a694eb-cb0e-4bce-ab68-aa417b054c63 /tidb/cdc/default/default/upstream/7365755495359373414 {"id":7365755495359373414,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49a694eb-cb0e-4bce-ab68-aa417b054c63 {"id":"49a694eb-cb0e-4bce-ab68-aa417b054c63","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973606} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c643f1cca 49a694eb-cb0e-4bce-ab68-aa417b054c63 /tidb/cdc/default/default/upstream/7365755495359373414 {"id":7365755495359373414,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ***************** properties ***************** "workload"="core" "updateproportion"="0" "requestdistribution"="uniform" "readallfields"="true" "mysql.db"="consistent_replicate_storage_file" "dotransactions"="false" "mysql.user"="root" "insertproportion"="0" "mysql.host"="127.0.0.1" "threadcount"="10" "readproportion"="0" "scanproportion"="0" "mysql.port"="4000" "recordcount"="5000" "operationcount"="0" "fieldcount"="100" ********************************************** table mark.finish_mark_3 exists table mark.finish_mark not exists for 1-th check, retry later owner pid: 5608 owner id 34206108-c200-4916-897e-35edd3bdd293 table test.availability1 exists nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_3 not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t50 exists check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } Run finished, takes 1.919098709s INSERT - Takes(s): 1.9, Count: 5000, OPS: 2617.5, Avg(us): 3480, Min(us): 1608, Max(us): 49489, 95th(us): 5000, 99th(us): 31000 [Pipeline] // stage [Pipeline] } table mark.finish_mark exists [Pipeline] // container 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0{"level":"warn","ts":"2024-05-06T13:33:31.104962+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001387500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:33:31.108366+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012ba000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:33:31.194792+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00132f6c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [Pipeline] } [Pipeline] // withEnv [Pipeline] } table mark.finish_mark_3 not exists for 4-th check, retry later [Pipeline] // node 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0{"level":"warn","ts":1714973611.5297728,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00075ec40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714973611.5298321,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714973611.5426474,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00399a000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714973611.5427222,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} [Pipeline] } [Pipeline] // podTemplate table consistent_replicate_storage_file.usertable exists table consistent_replicate_storage_file.check1 not exists for 1-th check, retry later [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully [Mon May 6 13:33:31 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.57585760.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_capture.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0{"level":"warn","ts":1714973612.259789,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002360380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714973612.2598634,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} check diff successfully wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... table mark.finish_mark_3 not exists for 5-th check, retry later table consistent_replicate_storage_file.check1 not exists for 2-th check, retry later wait process cdc.test exit for 3-th time... wait process cdc.test exit for 2-th time... wait process 8653 exit for 1-th time... wait process 8653 exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8653) - No such process wait process 8653 exit for 3-th time... process 8653 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 13:33:33 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.88218823.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:33:34 CST 2024] <<<<<< run test case default_value success! >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:33:34 CST 2024] <<<<<< run test case processor_etcd_worker_delay success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31925a78000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd, pid:16754, start at 2024-05-06 13:33:31.690334431 +0800 CST m=+5.768801145 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:35:31.698 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:33:31.678 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:23:31.678 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31925a78000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd, pid:16754, start at 2024-05-06 13:33:31.690334431 +0800 CST m=+5.768801145 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:35:31.698 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:33:31.678 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:23:31.678 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319253500015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd, pid:16837, start at 2024-05-06 13:33:31.259415003 +0800 CST m=+5.282836940 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:35:31.268 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:33:31.270 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:23:31.270 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31927a740007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd, pid:32927, start at 2024-05-06 13:33:33.732569241 +0800 CST m=+5.160324186 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:35:33.739 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:33:33.725 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:23:33.725 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31927a740007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd, pid:32927, start at 2024-05-06 13:33:33.732569241 +0800 CST m=+5.160324186 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:35:33.739 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:33:33.725 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:23:33.725 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:33:34 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b0b9e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570042914275341 MinTableBarrierTs:449570043176419345 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/34206108-c200-4916-897e-35edd3bdd293 {"id":"34206108-c200-4916-897e-35edd3bdd293","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973603} /tidb/cdc/default/__cdc_meta__/capture/47bfa9d2-6bbd-490c-a691-9c0210f4657e {"id":"47bfa9d2-6bbd-490c-a691-9c0210f4657e","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973612} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4587 34206108-c200-4916-897e-35edd3bdd293 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac45f8 47bfa9d2-6bbd-490c-a691-9c0210f4657e /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570042914275341,"min-table-barrier-ts":449570043176419345,"admin-job-type":0} /tidb/cdc/default/default/task/position/34206108-c200-4916-897e-35edd3bdd293/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/47bfa9d2-6bbd-490c-a691-9c0210f4657e/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b0b9e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570042914275341 MinTableBarrierTs:449570043176419345 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/34206108-c200-4916-897e-35edd3bdd293 {"id":"34206108-c200-4916-897e-35edd3bdd293","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973603} /tidb/cdc/default/__cdc_meta__/capture/47bfa9d2-6bbd-490c-a691-9c0210f4657e {"id":"47bfa9d2-6bbd-490c-a691-9c0210f4657e","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973612} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4587 34206108-c200-4916-897e-35edd3bdd293 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac45f8 47bfa9d2-6bbd-490c-a691-9c0210f4657e /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570042914275341,"min-table-barrier-ts":449570043176419345,"admin-job-type":0} /tidb/cdc/default/default/task/position/34206108-c200-4916-897e-35edd3bdd293/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/47bfa9d2-6bbd-490c-a691-9c0210f4657e/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b0b9e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570042914275341 MinTableBarrierTs:449570043176419345 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/34206108-c200-4916-897e-35edd3bdd293 {"id":"34206108-c200-4916-897e-35edd3bdd293","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973603} /tidb/cdc/default/__cdc_meta__/capture/47bfa9d2-6bbd-490c-a691-9c0210f4657e {"id":"47bfa9d2-6bbd-490c-a691-9c0210f4657e","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973612} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4587 34206108-c200-4916-897e-35edd3bdd293 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac45f8 47bfa9d2-6bbd-490c-a691-9c0210f4657e /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570042914275341,"min-table-barrier-ts":449570043176419345,"admin-job-type":0} /tidb/cdc/default/default/task/position/34206108-c200-4916-897e-35edd3bdd293/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/47bfa9d2-6bbd-490c-a691-9c0210f4657e/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "34206108-c200-4916-897e-35edd3bdd293" | grep id table mark.finish_mark_3 not exists for 6-th check, retry later table consistent_replicate_storage_file.check1 not exists for 3-th check, retry later "cluster-id": "default" "id": "47bfa9d2-6bbd-490c-a691-9c0210f4657e", "cluster-id": "default" run task successfully + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.18209.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:33:36 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/2eafe328-2689-494f-a8d8-cf680204606a {UpstreamID:7365755405094065760 Namespace:default ID:2eafe328-2689-494f-a8d8-cf680204606a SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:33:06.670803611 +0800 CST StartTs:449570035870203909 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003426ea0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570035896418311} {CheckpointTs:449570037757640734 MinTableBarrierTs:449570037757640734 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570037757640734, checkpointTs: 449570037757640734, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7b05b052-3504-4c99-99ed-9aeb378c2d76 {"id":"7b05b052-3504-4c99-99ed-9aeb378c2d76","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973613} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63efdb4a 7b05b052-3504-4c99-99ed-9aeb378c2d76 /tidb/cdc/default/default/changefeed/info/2eafe328-2689-494f-a8d8-cf680204606a {"upstream-id":7365755405094065760,"namespace":"default","changefeed-id":"2eafe328-2689-494f-a8d8-cf680204606a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:33:06.670803611+08:00","start-ts":449570035870203909,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570035896418311} /tidb/cdc/default/default/changefeed/status/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":449570037757640734,"min-table-barrier-ts":449570037757640734,"admin-job-type":0} /tidb/cdc/default/default/task/position/7b05b052-3504-4c99-99ed-9aeb378c2d76/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755405094065760 {"id":7365755405094065760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2eafe328-2689-494f-a8d8-cf680204606a {UpstreamID:7365755405094065760 Namespace:default ID:2eafe328-2689-494f-a8d8-cf680204606a SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:33:06.670803611 +0800 CST StartTs:449570035870203909 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003426ea0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570035896418311} {CheckpointTs:449570037757640734 MinTableBarrierTs:449570037757640734 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570037757640734, checkpointTs: 449570037757640734, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7b05b052-3504-4c99-99ed-9aeb378c2d76 {"id":"7b05b052-3504-4c99-99ed-9aeb378c2d76","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973613} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63efdb4a 7b05b052-3504-4c99-99ed-9aeb378c2d76 /tidb/cdc/default/default/changefeed/info/2eafe328-2689-494f-a8d8-cf680204606a {"upstream-id":7365755405094065760,"namespace":"default","changefeed-id":"2eafe328-2689-494f-a8d8-cf680204606a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:33:06.670803611+08:00","start-ts":449570035870203909,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570035896418311} /tidb/cdc/default/default/changefeed/status/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":449570037757640734,"min-table-barrier-ts":449570037757640734,"admin-job-type":0} /tidb/cdc/default/default/task/position/7b05b052-3504-4c99-99ed-9aeb378c2d76/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755405094065760 + grep -q 'failed to get info:' {"id":7365755405094065760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2eafe328-2689-494f-a8d8-cf680204606a {UpstreamID:7365755405094065760 Namespace:default ID:2eafe328-2689-494f-a8d8-cf680204606a SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:33:06.670803611 +0800 CST StartTs:449570035870203909 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003426ea0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570035896418311} {CheckpointTs:449570037757640734 MinTableBarrierTs:449570037757640734 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570037757640734, checkpointTs: 449570037757640734, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7b05b052-3504-4c99-99ed-9aeb378c2d76 {"id":"7b05b052-3504-4c99-99ed-9aeb378c2d76","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973613} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63efdb4a 7b05b052-3504-4c99-99ed-9aeb378c2d76 /tidb/cdc/default/default/changefeed/info/2eafe328-2689-494f-a8d8-cf680204606a {"upstream-id":7365755405094065760,"namespace":"default","changefeed-id":"2eafe328-2689-494f-a8d8-cf680204606a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:33:06.670803611+08:00","start-ts":449570035870203909,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570035896418311} /tidb/cdc/default/default/changefeed/status/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":449570037757640734,"min-table-barrier-ts":449570037757640734,"admin-job-type":0} /tidb/cdc/default/default/task/position/7b05b052-3504-4c99-99ed-9aeb378c2d76/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755405094065760 {"id":7365755405094065760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d31929b180015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-m22f9-nc6v2-09shd, pid:33008, start at 2024-05-06 13:33:35.850680374 +0800 CST m=+7.211068379 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:35:35.856 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:33:35.814 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:23:35.814 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/generate_column/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/generate_column/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 13:33:36.765 +08:00] [WARN] [diff.go:551] ["checksum is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] ["source checksum"=0] ["target checksum"=4294835404] ["get source checksum cost"=1.525883ms] ["get target checksum cost"=1.236583ms] 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0{"level":"warn","ts":"2024-05-06T13:33:37.106763+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001387500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:33:37.109607+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012ba000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:33:37.19572+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00132f6c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_reentrant/run.sh using Sink-Type: mysql... <<================= ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 7-th check, retry later table consistent_replicate_storage_file.check1 not exists for 4-th check, retry later nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + set +x + tso='449570043730853889 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570043730853889 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449570043730853889 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Mon May 6 13:33:37 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.1825018252.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.34324.out cli tso query --pd=http://127.0.0.1:2379 table mark.finish_mark_3 not exists for 8-th check, retry later nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later table consistent_replicate_storage_file.check1 not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_reentrant Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + set +x + tso='449570044438380546 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570044438380546 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:33:40 CST 2024] <<<<<< START cdc server in generate_column case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.3436534367.out server --log-file /tmp/tidb_cdc_test/generate_column/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/generate_column/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:33:41 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5bf00d2a-99c9-4491-9b44-aa08f2262716 {"id":"5bf00d2a-99c9-4491-9b44-aa08f2262716","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973618} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c64684ed2 5bf00d2a-99c9-4491-9b44-aa08f2262716 /tidb/cdc/default/default/upstream/7365755535292231898 {"id":7365755535292231898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5bf00d2a-99c9-4491-9b44-aa08f2262716 {"id":"5bf00d2a-99c9-4491-9b44-aa08f2262716","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973618} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c64684ed2 5bf00d2a-99c9-4491-9b44-aa08f2262716 /tidb/cdc/default/default/upstream/7365755535292231898 {"id":7365755535292231898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5bf00d2a-99c9-4491-9b44-aa08f2262716 {"id":"5bf00d2a-99c9-4491-9b44-aa08f2262716","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973618} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c64684ed2 5bf00d2a-99c9-4491-9b44-aa08f2262716 /tidb/cdc/default/default/upstream/7365755535292231898 {"id":7365755535292231898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449570043730853889 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.18315.out cli changefeed create --start-ts=449570043730853889 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365755535292231898,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T13:33:41.516576502+08:00","start_ts":449570043730853889,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570043730853889,"checkpoint_ts":449570043730853889,"checkpoint_time":"2024-05-06 13:33:36.527"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 exists table mark.finish_mark_4 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... table consistent_replicate_storage_file.check1 not exists for 6-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... {"level":"warn","ts":"2024-05-06T13:33:43.107268+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001387500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:33:43.110699+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012ba000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 3969 0 --:--:-- --:--:-- --:--:-- 4018 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 13:33:36.527","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-06 13:33:43.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '13:33:36.527","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '13:33:43.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '13:33:36.527","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '13:33:43.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-06 13:33:36.527' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '13:33:36.527","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '13:33:43.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '13:33:36.527","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '13:33:43.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-06 13:33:43' + echo 'sink_checkpoint_ts is 2024-05-06' 13:33:36.527 sink_checkpoint_ts is 2024-05-06 13:33:36.527 ++ date -d '2024-05-06 13:33:36.527' +%s + checkpoint_timestamp=1714973616 ++ date -d '2024-05-06 13:33:43' +%s + current_timestamp=1714973623 + '[' 7 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' {"level":"warn","ts":"2024-05-06T13:33:43.197176+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00132f6c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster table mark.finish_mark_4 not exists for 2-th check, retry later + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:33:43 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b6be117e-ebff-467f-88d5-c7e07df789c7 {"id":"b6be117e-ebff-467f-88d5-c7e07df789c7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973621} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c64768dd8 b6be117e-ebff-467f-88d5-c7e07df789c7 /tidb/cdc/default/default/upstream/7365755548449631840 {"id":7365755548449631840,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b6be117e-ebff-467f-88d5-c7e07df789c7 {"id":"b6be117e-ebff-467f-88d5-c7e07df789c7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973621} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c64768dd8 b6be117e-ebff-467f-88d5-c7e07df789c7 /tidb/cdc/default/default/upstream/7365755548449631840 {"id":7365755548449631840,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b6be117e-ebff-467f-88d5-c7e07df789c7 {"id":"b6be117e-ebff-467f-88d5-c7e07df789c7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973621} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c64768dd8 b6be117e-ebff-467f-88d5-c7e07df789c7 /tidb/cdc/default/default/upstream/7365755548449631840 {"id":7365755548449631840,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.34424.out cli changefeed create --start-ts=449570044438380546 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Create changefeed successfully! ID: 338c2100-f826-47ae-8baf-7ce0a9686fb3 Info: {"upstream_id":7365755548449631840,"namespace":"default","id":"338c2100-f826-47ae-8baf-7ce0a9686fb3","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:33:44.209842927+08:00","start_ts":449570044438380546,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570044438380546,"checkpoint_ts":449570044438380546,"checkpoint_time":"2024-05-06 13:33:39.226"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/sink_hang/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table consistent_replicate_storage_file.check1 not exists for 7-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x table mark.finish_mark_4 not exists for 3-th check, retry later table test.t1 exists + sleep 5 table consistent_replicate_storage_file.check1 not exists for 8-th check, retry later wait process 8826 exit for 1-th time... wait process 8826 exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8826) - No such process wait process 8826 exit for 3-th time... process 8826 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 13:33:46 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.89928994.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table mark.finish_mark_4 not exists for 4-th check, retry later table consistent_replicate_storage_file.check1 not exists for 9-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table generate_column.t exists table generate_column.t1 not exists for 1-th check, retry later table mark.finish_mark_4 not exists for 5-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:33:49 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/2eafe328-2689-494f-a8d8-cf680204606a {UpstreamID:7365755405094065760 Namespace:default ID:2eafe328-2689-494f-a8d8-cf680204606a SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:33:06.670803611 +0800 CST StartTs:449570035870203909 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0023ce090 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570035896418311} {CheckpointTs:449570037757640734 MinTableBarrierTs:449570037757640734 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570037757640734, checkpointTs: 449570037757640734, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d6c04df0-b58f-47d3-a0be-380a2988da5c {"id":"d6c04df0-b58f-47d3-a0be-380a2988da5c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973626} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63efdbc6 d6c04df0-b58f-47d3-a0be-380a2988da5c /tidb/cdc/default/default/changefeed/info/2eafe328-2689-494f-a8d8-cf680204606a {"upstream-id":7365755405094065760,"namespace":"default","changefeed-id":"2eafe328-2689-494f-a8d8-cf680204606a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:33:06.670803611+08:00","start-ts":449570035870203909,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570035896418311} /tidb/cdc/default/default/changefeed/status/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":449570037757640734,"min-table-barrier-ts":449570037757640734,"admin-job-type":0} /tidb/cdc/default/default/task/position/d6c04df0-b58f-47d3-a0be-380a2988da5c/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755405094065760 {"id":7365755405094065760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2eafe328-2689-494f-a8d8-cf680204606a {UpstreamID:7365755405094065760 Namespace:default ID:2eafe328-2689-494f-a8d8-cf680204606a SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:33:06.670803611 +0800 CST StartTs:449570035870203909 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0023ce090 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570035896418311} {CheckpointTs:449570037757640734 MinTableBarrierTs:449570037757640734 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570037757640734, checkpointTs: 449570037757640734, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d6c04df0-b58f-47d3-a0be-380a2988da5c {"id":"d6c04df0-b58f-47d3-a0be-380a2988da5c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973626} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63efdbc6 d6c04df0-b58f-47d3-a0be-380a2988da5c /tidb/cdc/default/default/changefeed/info/2eafe328-2689-494f-a8d8-cf680204606a {"upstream-id":7365755405094065760,"namespace":"default","changefeed-id":"2eafe328-2689-494f-a8d8-cf680204606a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:33:06.670803611+08:00","start-ts":449570035870203909,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570035896418311} /tidb/cdc/default/default/changefeed/status/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":449570037757640734,"min-table-barrier-ts":449570037757640734,"admin-job-type":0} /tidb/cdc/default/default/task/position/d6c04df0-b58f-47d3-a0be-380a2988da5c/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755405094065760 + grep -q 'failed to get info:' {"id":7365755405094065760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2eafe328-2689-494f-a8d8-cf680204606a {UpstreamID:7365755405094065760 Namespace:default ID:2eafe328-2689-494f-a8d8-cf680204606a SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:33:06.670803611 +0800 CST StartTs:449570035870203909 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0023ce090 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570035896418311} {CheckpointTs:449570037757640734 MinTableBarrierTs:449570037757640734 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570037757640734, checkpointTs: 449570037757640734, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d6c04df0-b58f-47d3-a0be-380a2988da5c {"id":"d6c04df0-b58f-47d3-a0be-380a2988da5c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973626} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63efdbc6 d6c04df0-b58f-47d3-a0be-380a2988da5c /tidb/cdc/default/default/changefeed/info/2eafe328-2689-494f-a8d8-cf680204606a {"upstream-id":7365755405094065760,"namespace":"default","changefeed-id":"2eafe328-2689-494f-a8d8-cf680204606a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:33:06.670803611+08:00","start-ts":449570035870203909,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570035896418311} /tidb/cdc/default/default/changefeed/status/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":449570037757640734,"min-table-barrier-ts":449570037757640734,"admin-job-type":0} /tidb/cdc/default/default/task/position/d6c04df0-b58f-47d3-a0be-380a2988da5c/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755405094065760 {"id":7365755405094065760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully table consistent_replicate_storage_file.check1 not exists for 10-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=2 and val=2 run task successfully wait process cdc.test exit for 1-th time... table generate_column.t1 exists check diff failed 1-th time, retry later wait process cdc.test exit for 2-th time... + run_case_with_unavailable_tikv conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_hang_up_capture [Mon May 6 13:33:51 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.59595961.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 1475 0 --:--:-- --:--:-- --:--:-- 1481 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 13:33:49.978","puller_resolved_ts":"2024-05-06 13:33:43.178","last_synced_ts":"2024-05-06 13:33:43.678","now_ts":"2024-05-06 13:33:50.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '13:33:49.978","puller_resolved_ts":"2024-05-06' '13:33:43.178","last_synced_ts":"2024-05-06' '13:33:43.678","now_ts":"2024-05-06' '13:33:50.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '13:33:49.978","puller_resolved_ts":"2024-05-06' '13:33:43.178","last_synced_ts":"2024-05-06' '13:33:43.678","now_ts":"2024-05-06' '13:33:50.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 table mark.finish_mark_4 not exists for 6-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table consistent_replicate_storage_file.check1 not exists for 11-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319392340014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-855t7-fbskf-4lwp5, pid:7082, start at 2024-05-06 13:33:51.672739578 +0800 CST m=+5.296140021 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:35:51.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:33:51.679 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:23:51.679 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully wait process cdc.test exit for 1-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... table mark.finish_mark_4 not exists for 7-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/sink_hang Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:33:54 CST 2024] <<<<<< run test case generate_column success! >>>>>> table consistent_replicate_storage_file.check1 not exists for 12-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319392340014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-855t7-fbskf-4lwp5, pid:7082, start at 2024-05-06 13:33:51.672739578 +0800 CST m=+5.296140021 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:35:51.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:33:51.679 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:23:51.679 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319393ac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-855t7-fbskf-4lwp5, pid:7153, start at 2024-05-06 13:33:51.750910526 +0800 CST m=+5.311504364 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:35:51.758 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:33:51.723 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:23:51.723 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_reentrant/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_reentrant/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:33:54 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001450360 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570048157155336 MinTableBarrierTs:449570048170000387 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570048170000387, checkpointTs: 449570048170000387, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570048157155336, checkpointTs: 449570048157155336, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570048170000387, checkpointTs: 449570048170000387, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/db77c40d-4e6e-4643-98c8-1376add879ba {"id":"db77c40d-4e6e-4643-98c8-1376add879ba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973631} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac46b4 db77c40d-4e6e-4643-98c8-1376add879ba /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570048157155336,"min-table-barrier-ts":449570048170000387,"admin-job-type":0} /tidb/cdc/default/default/task/position/db77c40d-4e6e-4643-98c8-1376add879ba/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001450360 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570048157155336 MinTableBarrierTs:449570048170000387 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570048170000387, checkpointTs: 449570048170000387, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570048157155336, checkpointTs: 449570048157155336, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570048170000387, checkpointTs: 449570048170000387, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/db77c40d-4e6e-4643-98c8-1376add879ba {"id":"db77c40d-4e6e-4643-98c8-1376add879ba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973631} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac46b4 db77c40d-4e6e-4643-98c8-1376add879ba /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570048157155336,"min-table-barrier-ts":449570048170000387,"admin-job-type":0} /tidb/cdc/default/default/task/position/db77c40d-4e6e-4643-98c8-1376add879ba/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001450360 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570048157155336 MinTableBarrierTs:449570048170000387 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570048170000387, checkpointTs: 449570048170000387, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570048157155336, checkpointTs: 449570048157155336, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570048170000387, checkpointTs: 449570048170000387, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/db77c40d-4e6e-4643-98c8-1376add879ba {"id":"db77c40d-4e6e-4643-98c8-1376add879ba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973631} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac46b4 db77c40d-4e6e-4643-98c8-1376add879ba /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570048157155336,"min-table-barrier-ts":449570048170000387,"admin-job-type":0} /tidb/cdc/default/default/task/position/db77c40d-4e6e-4643-98c8-1376add879ba/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... table mark.finish_mark_4 not exists for 8-th check, retry later "is-owner": true, run task successfully table consistent_replicate_storage_file.check1 not exists for 13-th check, retry later skip some DDLs in tidb v4.0.x [Mon May 6 13:33:56 CST 2024] <<<<<< START cdc server in ddl_reentrant case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_reentrant.83998401.out server --log-file /tmp/tidb_cdc_test/ddl_reentrant/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_reentrant/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) owner pid: 5964 owner id db77c40d-4e6e-4643-98c8-1376add879ba [Mon May 6 13:33:57 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.60836085.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_capture.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table mark.finish_mark_4 not exists for 9-th check, retry later wait process 8997 exit for 1-th time... table consistent_replicate_storage_file.check1 not exists for 14-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process 8997 exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8997) - No such process wait process 8997 exit for 3-th time... process 8997 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 13:33:59 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.91699171.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table mark.finish_mark_4 exists table mark.finish_mark not exists for 1-th check, retry later < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:33:59 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/01c1c369-a3bf-45d5-a0ea-004e7e59db0a {"id":"01c1c369-a3bf-45d5-a0ea-004e7e59db0a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973637} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c64b832cc 01c1c369-a3bf-45d5-a0ea-004e7e59db0a /tidb/cdc/default/default/upstream/7365755627546970540 {"id":7365755627546970540,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/01c1c369-a3bf-45d5-a0ea-004e7e59db0a {"id":"01c1c369-a3bf-45d5-a0ea-004e7e59db0a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973637} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c64b832cc 01c1c369-a3bf-45d5-a0ea-004e7e59db0a /tidb/cdc/default/default/upstream/7365755627546970540 {"id":7365755627546970540,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/01c1c369-a3bf-45d5-a0ea-004e7e59db0a {"id":"01c1c369-a3bf-45d5-a0ea-004e7e59db0a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973637} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c64b832cc 01c1c369-a3bf-45d5-a0ea-004e7e59db0a /tidb/cdc/default/default/upstream/7365755627546970540 {"id":7365755627546970540,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ------------------------------------------ test ddl create database ddl_reentrant, is_reentrant: false restored_sql: CREATE DATABASE `ddl_reentrant` check_ts_forward 9d8cdff7-a113-4d42-a709-c50165c66c5f Starting Upstream TiDB... Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file.check1 not exists for 15-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:34:00 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e2c360 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570049743126531 MinTableBarrierTs:449570049743126533 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ef1decc-b230-46b0-944f-7f1751c7e2d9 {"id":"6ef1decc-b230-46b0-944f-7f1751c7e2d9","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973637} /tidb/cdc/default/__cdc_meta__/capture/db77c40d-4e6e-4643-98c8-1376add879ba {"id":"db77c40d-4e6e-4643-98c8-1376add879ba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973631} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac46b4 db77c40d-4e6e-4643-98c8-1376add879ba /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4705 6ef1decc-b230-46b0-944f-7f1751c7e2d9 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570049743126531,"min-table-barrier-ts":449570049743126533,"admin-job-type":0} /tidb/cdc/default/default/task/position/6ef1decc-b230-46b0-944f-7f1751c7e2d9/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/db77c40d-4e6e-4643-98c8-1376add879ba/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e2c360 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570049743126531 MinTableBarrierTs:449570049743126533 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ef1decc-b230-46b0-944f-7f1751c7e2d9 {"id":"6ef1decc-b230-46b0-944f-7f1751c7e2d9","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973637} /tidb/cdc/default/__cdc_meta__/capture/db77c40d-4e6e-4643-98c8-1376add879ba {"id":"db77c40d-4e6e-4643-98c8-1376add879ba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973631} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac46b4 db77c40d-4e6e-4643-98c8-1376add879ba /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4705 6ef1decc-b230-46b0-944f-7f1751c7e2d9 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570049743126531,"min-table-barrier-ts":449570049743126533,"admin-job-type":0} /tidb/cdc/default/default/task/position/6ef1decc-b230-46b0-944f-7f1751c7e2d9/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkp+ grep -q 'failed to get info:' oint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/db77c40d-4e6e-4643-98c8-1376add879ba/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e2c360 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570049743126531 MinTableBarrierTs:449570049743126533 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ef1decc-b230-46b0-944f-7f1751c7e2d9 {"id":"6ef1decc-b230-46b0-944f-7f1751c7e2d9","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973637} /tidb/cdc/default/__cdc_meta__/capture/db77c40d-4e6e-4643-98c8-1376add879ba {"id":"db77c40d-4e6e-4643-98c8-1376add879ba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973631} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac46b4 db77c40d-4e6e-4643-98c8-1376add879ba /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4705 6ef1decc-b230-46b0-944f-7f1751c7e2d9 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570049743126531,"min-table-barrier-ts":449570049743126533,"admin-job-type":0} /tidb/cdc/default/default/task/position/6ef1decc-b230-46b0-944f-7f1751c7e2d9/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/db77c40d-4e6e-4643-98c8-1376add879ba/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "db77c40d-4e6e-4643-98c8-1376add879ba" | grep id Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3193fb18000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:22191, start at 2024-05-06 13:33:58.35356874 +0800 CST m=+5.221107351 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:35:58.363 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:33:58.342 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:23:58.342 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3193fb18000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:22191, start at 2024-05-06 13:33:58.35356874 +0800 CST m=+5.221107351 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:35:58.363 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:33:58.342 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:23:58.342 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3193fc9c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:22267, start at 2024-05-06 13:33:58.449246831 +0800 CST m=+5.270078148 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:35:58.455 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:33:58.439 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:23:58.439 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/simple/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/simple/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/simple/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table mark.finish_mark not exists for 2-th check, retry later "cluster-id": "default" "id": "6ef1decc-b230-46b0-944f-7f1751c7e2d9", "cluster-id": "default" run task successfully changefeed is working normally rts: 449570049839595500->449570050258763800 checkpoint: 449570049839595500->449570050258763800 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/06 13:34:01.671 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="CREATE DATABASE `ddl_reentrant`"] [duration=78.674106ms] [namespace=default] [changefeed=9d8cdff7-a113-4d42-a709-c50165c66c5f] run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:34:02 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/2eafe328-2689-494f-a8d8-cf680204606a {UpstreamID:7365755405094065760 Namespace:default ID:2eafe328-2689-494f-a8d8-cf680204606a SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:33:06.670803611 +0800 CST StartTs:449570035870203909 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a93c20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570035896418311} {CheckpointTs:449570037757640734 MinTableBarrierTs:449570037783855134 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570037757640734, checkpointTs: 449570037757640734, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570037757640734, checkpointTs: 449570037757640734, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/61871c6e-ae16-4270-bdfd-13b7cc7c1af5 {"id":"61871c6e-ae16-4270-bdfd-13b7cc7c1af5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973639} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63efdc38 61871c6e-ae16-4270-bdfd-13b7cc7c1af5 /tidb/cdc/default/default/changefeed/info/2eafe328-2689-494f-a8d8-cf680204606a {"upstream-id":7365755405094065760,"namespace":"default","changefeed-id":"2eafe328-2689-494f-a8d8-cf680204606a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:33:06.670803611+08:00","start-ts":449570035870203909,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570035896418311} /tidb/cdc/default/default/changefeed/status/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":449570037757640734,"min-table-barrier-ts":449570037783855134,"admin-job-type":0} /tidb/cdc/default/default/task/position/61871c6e-ae16-4270-bdfd-13b7cc7c1af5/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755405094065760 {"id":7365755405094065760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2eafe328-2689-494f-a8d8-cf680204606a {UpstreamID:7365755405094065760 Namespace:default ID:2eafe328-2689-494f-a8d8-cf680204606a SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:33:06.670803611 +0800 CST StartTs:449570035870203909 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a93c20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570035896418311} {CheckpointTs:449570037757640734 MinTableBarrierTs:449570037783855134 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570037757640734, checkpointTs: 449570037757640734, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570037757640734, checkpointTs: 449570037757640734, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/61871c6e-ae16-4270-bdfd-13b7cc7c1af5 {"id":"61871c6e-ae16-4270-bdfd-13b7cc7c1af5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973639} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63efdc38 61871c6e-ae16-4270-bdfd-13b7cc7c1af5 /tidb/cdc/default/default/changefeed/info/2eafe328-2689-494f-a8d8-cf680204606a {"upstream-id":7365755405094065760,"namespace":"default","changefeed-id":"2eafe328-2689-494f-a8d8-cf680204606a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:33:06.670803611+08:00","start-ts":449570035870203909,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570035896418311} /tidb/cdc/default/default/changefeed/status/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":449570037757640734,"min-table-barrier-ts":449570037783855134,"admin-job-type":0} /tidb/cdc/default/default/task/position/61871c6e-ae16-4270-bdfd-13b7cc7c1af5/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755405094065760 {"id":7365755405094065760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2eafe328-2689-494f-a8d8-cf680204606a {UpstreamID:7365755405094065760 Namespace:default ID:2eafe328-2689-494f-a8d8-cf680204606a SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:33:06.670803611 +0800 CST StartTs:449570035870203909 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a93c20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570035896418311} {CheckpointTs:449570037757640734 MinTableBarrierTs:449570037783855134 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570037757640734, checkpointTs: 449570037757640734, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570037757640734, checkpointTs: 449570037757640734, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/61871c6e-ae16-4270-bdfd-13b7cc7c1af5 {"id":"61871c6e-ae16-4270-bdfd-13b7cc7c1af5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973639} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c63efdc38 61871c6e-ae16-4270-bdfd-13b7cc7c1af5 /tidb/cdc/default/default/changefeed/info/2eafe328-2689-494f-a8d8-cf680204606a {"upstream-id":7365755405094065760,"namespace":"default","changefeed-id":"2eafe328-2689-494f-a8d8-cf680204606a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:33:06.670803611+08:00","start-ts":449570035870203909,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570035896418311} /tidb/cdc/default/default/changefeed/status/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":449570037757640734,"min-table-barrier-ts":449570037783855134,"admin-job-type":0} /tidb/cdc/default/default/task/position/61871c6e-ae16-4270-bdfd-13b7cc7c1af5/2eafe328-2689-494f-a8d8-cf680204606a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755405094065760 {"id":7365755405094065760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully table kill_owner_with_ddl.t1 exists table kill_owner_with_ddl.t2 exists table kill_owner_with_ddl.t3 not exists for 1-th check, retry later table consistent_replicate_storage_file.check1 not exists for 16-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) resume changefeed 9d8cdff7-a113-4d42-a709-c50165c66c5f from 449570049892024321 check_ts_forward 9d8cdff7-a113-4d42-a709-c50165c66c5f nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.23574.out cli tso query --pd=http://127.0.0.1:2379 table mark.finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file.check1 not exists for 17-th check, retry later table kill_owner_with_ddl.t3 exists check diff failed 1-th time, retry later changefeed is working normally rts: 449570049892024300->449570049892024300 checkpoint: 449570049892024300->449570051045457900 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2024/05/06 13:34:04.592 +08:00] [INFO] [mysql_ddl_sink.go:127] ["Execute DDL failed, but error can be ignored"] [startTs=449570049878917147] [ddl="CREATE DATABASE `ddl_reentrant`"] [namespace=default] [changefeed=9d8cdff7-a113-4d42-a709-c50165c66c5f] [error="Error 1007 (HY000): Can't create database 'ddl_reentrant'; database exists"] run task successfully ------------------------------------------ test ddl create table ddl_reentrant.t3 (a int primary key, b int) partition by range(a) (partition p0 values less than (1000), partition p1 values less than (2000)), is_reentrant: false restored_sql: CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000)) check_ts_forward 9d8cdff7-a113-4d42-a709-c50165c66c5f ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449570050865364993 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570050865364993 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later table mark.finish_mark not exists for 4-th check, retry later table consistent_replicate_storage_file.check1 exists table consistent_replicate_storage_file.t2 not exists for 1-th check, retry later [Mon May 6 13:34:06 CST 2024] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.2362323625.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319473440013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57, pid:12619, start at 2024-05-06 13:34:06.082934996 +0800 CST m=+5.477293505 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:06.092 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:06.083 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:06.083 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } wait process cdc.test exit for 1-th time... [Pipeline] // node changefeed is working normally rts: 449570051045457900->449570051163422700 checkpoint: 449570051045457900->449570051163422700 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/06 13:34:05.573 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000))"] [duration=80.091642ms] [namespace=default] [changefeed=9d8cdff7-a113-4d42-a709-c50165c66c5f] run task successfully [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv wait process cdc.test exit for 2-th time... [Pipeline] } [Pipeline] // stage resume changefeed 9d8cdff7-a113-4d42-a709-c50165c66c5f from 449570051163422736 check_ts_forward 9d8cdff7-a113-4d42-a709-c50165c66c5f [Pipeline] } table mark.finish_mark not exists for 5-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:34:08 CST 2024] <<<<<< run test case kill_owner_with_ddl success! >>>>>> table consistent_replicate_storage_file.t2 not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319466980017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-l2sg4-3h871-3slfr, pid:7645, start at 2024-05-06 13:34:05.246677295 +0800 CST m=+5.198304048 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:05.254 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:05.222 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:05.222 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319466980017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-l2sg4-3h871-3slfr, pid:7645, start at 2024-05-06 13:34:05.246677295 +0800 CST m=+5.198304048 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:05.254 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:05.222 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:05.222 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319467500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-l2sg4-3h871-3slfr, pid:7729, start at 2024-05-06 13:34:05.291886619 +0800 CST m=+5.164372420 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:05.299 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:05.268 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:05.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_hang/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_hang/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319473440013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57, pid:12619, start at 2024-05-06 13:34:06.082934996 +0800 CST m=+5.477293505 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:06.092 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:06.083 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:06.083 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319473f40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57, pid:12699, start at 2024-05-06 13:34:06.120550686 +0800 CST m=+5.445458108 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:06.130 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:06.126 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:06.126 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:34:09 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50be4725-f10f-4985-8f34-654bfe6fcee5 {"id":"50be4725-f10f-4985-8f34-654bfe6fcee5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973646} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c64d29ef5 50be4725-f10f-4985-8f34-654bfe6fcee5 /tidb/cdc/default/default/upstream/7365755654744004501 {"id":7365755654744004501,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50be4725-f10f-4985-8f34-654bfe6fcee5 {"id":"50be4725-f10f-4985-8f34-654bfe6fcee5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973646} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c64d29ef5 50be4725-f10f-4985-8f34-654bfe6fcee5 /tidb/cdc/default/default/upstream/7365755654744004501 {"id":7365755654744004501,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50be4725-f10f-4985-8f34-654bfe6fcee5 {"id":"50be4725-f10f-4985-8f34-654bfe6fcee5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973646} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c64d29ef5 50be4725-f10f-4985-8f34-654bfe6fcee5 /tidb/cdc/default/default/upstream/7365755654744004501 {"id":7365755654744004501,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.23688.out cli changefeed create --start-ts=449570050865364993 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ table mark.finish_mark not exists for 6-th check, retry later Create changefeed successfully! ID: 88f364ac-9472-4c90-9b4e-ba2b4615be9d Info: {"upstream_id":7365755654744004501,"namespace":"default","id":"88f364ac-9472-4c90-9b4e-ba2b4615be9d","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:34:09.819644525+08:00","start_ts":449570050865364993,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570050865364993,"checkpoint_ts":449570050865364993,"checkpoint_time":"2024-05-06 13:34:03.743"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later run task failed 1-th time, retry later table consistent_replicate_storage_file.t2 not exists for 3-th check, retry later + set +x succeed to verify meta placement rules [Mon May 6 13:34:10 CST 2024] <<<<<< START cdc server in sink_hang case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=2*return(true)' + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_hang.90559057.out server --log-file /tmp/tidb_cdc_test/sink_hang/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_hang/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.13947.out cli tso query --pd=http://127.0.0.1:2379 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later pass check, checkpoint tso not forward after 10s run task successfully wait process 8044 exit for 1-th time... wait process 8044 exit for 2-th time... wait process 8044 exit for 3-th time... wait process 8044 exit for 4-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8044) - No such process wait process 8044 exit for 5-th time... process 8044 already exit [Mon May 6 13:34:05 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.87948796.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:34:08 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365755301910702846 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 13:32:44.395107764 +0800 CST StartTs:449570030036189189 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0026c7290 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570030062403586} {CheckpointTs:449570033994563594 MinTableBarrierTs:449570052174249986 AdminJobType:noop} span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449570033994563594, checkpointTs: 449570033994563594, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570033994563594, checkpointTs: 449570033994563594, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570033994563594, checkpointTs: 449570033994563594, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449570033994563594, checkpointTs: 449570033994563594, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c0c339f-8853-40c4-9622-f2eb178065e7 {"id":"7c0c339f-8853-40c4-9622-f2eb178065e7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973646} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c639023c7 7c0c339f-8853-40c4-9622-f2eb178065e7 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365755301910702846,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T13:32:44.395107764+08:00","start-ts":449570030036189189,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570030062403586} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449570033994563594,"min-table-barrier-ts":449570052174249986,"admin-job-type":0} /tidb/cdc/default/default/task/position/7c0c339f-8853-40c4-9622-f2eb178065e7/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755301910702846 {"id":7365755301910702846,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365755301910702846 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 13:32:44.395107764 +0800 CST StartTs:449570030036189189 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0026c7290 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570030062403586} {CheckpointTs:449570033994563594 MinTableBarrierTs:449570052174249986 AdminJobType:noop} span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449570033994563594, checkpointTs: 449570033994563594, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570033994563594, checkpointTs: 449570033994563594, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570033994563594, checkpointTs: 449570033994563594, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449570033994563594, checkpointTs: 449570033994563594, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c0c339f-8853-40c4-9622-f2eb178065e7 {"id":"7c0c339f-8853-40c4-9622-f2eb178065e7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973646} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c639023c7 7c0c339f-8853-40c4-9622-f2eb178065e7 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365755301910702846,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T13:32:44.395107764+08:00","start-ts":449570030036189189,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570030062403586} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449570033994563594,"min-table-barrier-ts":449570052174249986,"admin-job-type":0} /tidb/cdc/default/default/task/position/7c0c339f-8853-40c4-9622-f2eb178065e7/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755301910702846 {"id":7365755301910702846,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365755301910702846 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 13:32:44.395107764 +0800 CST StartTs:449570030036189189 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0026c7290 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570030062403586} {CheckpointTs:449570033994563594 MinTableBarrierTs:449570052174249986 AdminJobType:noop} span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449570033994563594, checkpointTs: 449570033994563594, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570033994563594, checkpointTs: 449570033994563594, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570033994563594, checkpointTs: 449570033994563594, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449570033994563594, checkpointTs: 449570033994563594, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c0c339f-8853-40c4-9622-f2eb178065e7 {"id":"7c0c339f-8853-40c4-9622-f2eb178065e7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973646} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c639023c7 7c0c339f-8853-40c4-9622-f2eb178065e7 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365755301910702846,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T13:32:44.395107764+08:00","start-ts":449570030036189189,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570030062403586} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkp+ grep -q 'etcd info' oint-ts":449570033994563594,"min-table-barrier-ts":449570052174249986,"admin-job-type":0} /tidb/cdc/default/default/task/position/7c0c339f-8853-40c4-9622-f2eb178065e7/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755301910702846 {"id":7365755301910702846,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check diff successfully check_ts_forward ddl-only-block-related-table changefeed is working normally rts: 449570033994563600->449570052436394000 checkpoint: 449570033994563600->449570052698538000 run task successfully wait process cdc.test exit for 1-th time... table mark.finish_mark not exists for 7-th check, retry later check_ts_forward 9d8cdff7-a113-4d42-a709-c50165c66c5f wait process cdc.test exit for 2-th time... table consistent_replicate_storage_file.t2 not exists for 4-th check, retry later + set +x + tso='449570052841930753 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570052841930753 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449570052841930753 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Mon May 6 13:34:12 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1398713989.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:34:12 CST 2024] <<<<<< run test case ddl_only_block_related_table success! >>>>>> pass test: get status pass test: health pass test: changefeed apis pass test: delete changefeed apis pass test: capture apis pass test: processor apis pass test: owner apis pass test: set log level + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit ++ date + echo '[Mon May 6 13:34:06 CST 2024] <<<<<< run test case api_v2 success! >>>>>>' [Mon May 6 13:34:06 CST 2024] <<<<<< run test case api_v2 success! >>>>>> + stop_tidb_cluster table mark.finish_mark exists check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > changefeed is working normally rts: 449570052893311000->449570053155717100 checkpoint: 449570052893311000->449570053155717100 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2024/05/06 13:34:09.594 +08:00] [INFO] [mysql_ddl_sink.go:127] ["Execute DDL failed, but error can be ignored"] [startTs=449570051163422729] [ddl="CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000))"] [namespace=default] [changefeed=9d8cdff7-a113-4d42-a709-c50165c66c5f] [error="Error 1050 (42S01): Table 'ddl_reentrant.t3' already exists"] run task successfully ------------------------------------------ test ddl create table ddl_reentrant.t4 (a int primary key, b int), is_reentrant: false restored_sql: CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT) check_ts_forward 9d8cdff7-a113-4d42-a709-c50165c66c5f TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:34:14 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1b596e61-b25d-4fe2-b399-805cb27a8dfd {"id":"1b596e61-b25d-4fe2-b399-805cb27a8dfd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973651} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c64ed7fd5 1b596e61-b25d-4fe2-b399-805cb27a8dfd /tidb/cdc/default/default/upstream/7365755674121732095 {"id":7365755674121732095,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1b596e61-b25d-4fe2-b399-805cb27a8dfd {"id":"1b596e61-b25d-4fe2-b399-805cb27a8dfd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973651} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c64ed7fd5 1b596e61-b25d-4fe2-b399-805cb27a8dfd /tidb/cdc/default/default/upstream/7365755674121732095 {"id":7365755674121732095,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1b596e61-b25d-4fe2-b399-805cb27a8dfd {"id":"1b596e61-b25d-4fe2-b399-805cb27a8dfd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973651} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c64ed7fd5 1b596e61-b25d-4fe2-b399-805cb27a8dfd /tidb/cdc/default/default/upstream/7365755674121732095 {"id":7365755674121732095,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table consistent_replicate_storage_file.t2 not exists for 5-th check, retry later wait process cdc.test exit for 1-th time... check_changefeed_status 127.0.0.1:8300 42a14419-b7ec-46b8-9a18-d4ad96ba4dad normal last_error null + endpoint=127.0.0.1:8300 + changefeed_id=42a14419-b7ec-46b8-9a18-d4ad96ba4dad + expected_state=normal + field=last_error + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/42a14419-b7ec-46b8-9a18-d4ad96ba4dad/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 801 0 --:--:-- --:--:-- --:--:-- 803 + info='{"state":"normal","resolved_ts":449570053586157571,"checkpoint_ts":449570053586157571}' + echo '{"state":"normal","resolved_ts":449570053586157571,"checkpoint_ts":449570053586157571}' {"state":"normal","resolved_ts":449570053586157571,"checkpoint_ts":449570053586157571} ++ echo '{"state":"normal","resolved_ts":449570053586157571,"checkpoint_ts":449570053586157571}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_error ]] ++ echo '{"state":"normal","resolved_ts":449570053586157571,"checkpoint_ts":449570053586157571}' ++ jq -r .last_error.message + error_msg=null + [[ ! null =~ null ]] run task successfully check_changefeed_status 127.0.0.1:8300 42a14419-b7ec-46b8-9a18-d4ad96ba4dad normal last_warning null + endpoint=127.0.0.1:8300 + changefeed_id=42a14419-b7ec-46b8-9a18-d4ad96ba4dad + expected_state=normal + field=last_warning + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/42a14419-b7ec-46b8-9a18-d4ad96ba4dad/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 669 0 --:--:-- --:--:-- --:--:-- 671 + info='{"state":"normal","resolved_ts":449570053586157571,"checkpoint_ts":449570053586157571}' + echo '{"state":"normal","resolved_ts":449570053586157571,"checkpoint_ts":449570053586157571}' {"state":"normal","resolved_ts":449570053586157571,"checkpoint_ts":449570053586157571} ++ echo '{"state":"normal","resolved_ts":449570053586157571,"checkpoint_ts":449570053586157571}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_warning ]] ++ echo '{"state":"normal","resolved_ts":449570053586157571,"checkpoint_ts":449570053586157571}' ++ jq -r .last_warning.message + error_msg=null + [[ ! null =~ null ]] run task successfully table sink_hang.t1 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:34:15 CST 2024] <<<<<< run test case multi_source success! >>>>>> changefeed is working normally rts: 449570053417861100->449570053417861100 checkpoint: 449570053417861100->449570053496242200 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/06 13:34:15.823 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT)"] [duration=79.766115ms] [namespace=default] [changefeed=9d8cdff7-a113-4d42-a709-c50165c66c5f] run task successfully check data successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:34:15 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc1ca19a-b158-4fd8-b56d-d6b54ed5359a {"id":"fc1ca19a-b158-4fd8-b56d-d6b54ed5359a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973653} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c64efadd6 fc1ca19a-b158-4fd8-b56d-d6b54ed5359a /tidb/cdc/default/default/upstream/7365755677243832462 {"id":7365755677243832462,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc1ca19a-b158-4fd8-b56d-d6b54ed5359a {"id":"fc1ca19a-b158-4fd8-b56d-d6b54ed5359a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973653} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c64efadd6 fc1ca19a-b158-4fd8-b56d-d6b54ed5359a /tidb/cdc/default/default/upstream/7365755677243832462 {"id":7365755677243832462,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc1ca19a-b158-4fd8-b56d-d6b54ed5359a {"id":"fc1ca19a-b158-4fd8-b56d-d6b54ed5359a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973653} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c64efadd6 fc1ca19a-b158-4fd8-b56d-d6b54ed5359a /tidb/cdc/default/default/upstream/7365755677243832462 {"id":7365755677243832462,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449570052841930753 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.14037.out cli changefeed create --start-ts=449570052841930753 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml wait process cdc.test exit for 1-th time... table consistent_replicate_storage_file.t2 not exists for 6-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/http_api_tls_with_user_auth/run.sh using Sink-Type: mysql... <<================= Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365755677243832462,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T13:34:16.336399449+08:00","start_ts":449570052841930753,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570052841930753,"checkpoint_ts":449570052841930753,"checkpoint_time":"2024-05-06 13:34:11.283"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... resume changefeed 9d8cdff7-a113-4d42-a709-c50165c66c5f from 449570053496242193 check_ts_forward 9d8cdff7-a113-4d42-a709-c50165c66c5f nonempty select id, val from test.availability1 where id=3 and val=3 run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... table sink_hang.t1 exists table sink_hang.t2 not exists for 1-th check, retry later wait process cdc.test exit for 3-th time... WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Requirement already up-to-date: requests==2.26.0 in /usr/local/lib/python3.6/site-packages wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:34:17 CST 2024] <<<<<< run test case simple success! >>>>>> Requirement already up-to-date: idna<4,>=2.5; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit run test case test_expire_capture [Mon May 6 13:34:17 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.62916293.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: certifi>=2017.4.17 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) The 1 times to try to start tidb cluster... + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/owner_remove_table_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table consistent_replicate_storage_file.t2 exists check diff successfully run task failed 1-th time, retry later wait process cdc.test exit for 1-th time... table sink_hang.t2 exists check diff failed 1-th time, retry later wait process cdc.test exit for 2-th time... table test.t1 exists + sleep 5 wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:34:20 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file.1151811520.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check_ts_forward 9d8cdff7-a113-4d42-a709-c50165c66c5f + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:34:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00149b7a0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570054986006532 MinTableBarrierTs:449570055248150534 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570055248150532, checkpointTs: 449570055248150532, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570055248150534, checkpointTs: 449570055248150534, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570055248150534, checkpointTs: 449570055248150534, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6fa32d8a-a720-4a9f-b696-0a2dde70aed5 {"id":"6fa32d8a-a720-4a9f-b696-0a2dde70aed5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973657} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac47d2 6fa32d8a-a720-4a9f-b696-0a2dde70aed5 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570054986006534,"min-table-barrier-ts":449570055248150534,"admin-job-type":0} /tidb/cdc/default/default/task/position/6fa32d8a-a720-4a9f-b696-0a2dde70aed5/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00149b7a0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570054986006532 MinTableBarrierTs:449570055248150534 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570055248150532, checkpointTs: 449570055248150532, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570055248150534, checkpointTs: 449570055248150534, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570055248150534, checkpointTs: 449570055248150534, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6fa32d8a-a720-4a9f-b696-0a2dde70aed5 {"id":"6fa32d8a-a720-4a9f-b696-0a2dde70aed5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973657} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac47d2 6fa32d8a-a720-4a9f-b696-0a2dde70aed5 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570054986006534,"min-table-barrier-ts":449570055248150534,"admin-job-type":0} /tidb/cdc/default/default/task/position/6fa32d8a-a720-4a9f-b696-0a2dde70aed5/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00149b7a0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570054986006532 MinTableBarrierTs:449570055248150534 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570055248150532, checkpointTs: 449570055248150532, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570055248150534, checkpointTs: 449570055248150534, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570055248150534, checkpointTs: 449570055248150534, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6fa32d8a-a720-4a9f-b696-0a2dde70aed5 {"id":"6fa32d8a-a720-4a9f-b696-0a2dde70aed5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973657} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac47d2 6fa32d8a-a720-4a9f-b696-0a2dde70aed5 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570054986006534,"min-table-barrier-ts":449570055248150534,"admin-job-type":0} /tidb/cdc/default/default/task/position/6fa32d8a-a720-4a9f-b696-0a2dde70aed5/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' start tidb cluster in /tmp/tidb_cdc_test/http_api_tls_with_user_auth Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... check diff failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/owner_remove_table_error Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... "is-owner": true, run task successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release changefeed is working normally rts: 449570055252869100->449570055514751000 checkpoint: 449570055252869100->449570055514751000 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2024/05/06 13:34:18.593 +08:00] [INFO] [mysql_ddl_sink.go:127] ["Execute DDL failed, but error can be ignored"] [startTs=449570053496242187] [ddl="CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT)"] [namespace=default] [changefeed=9d8cdff7-a113-4d42-a709-c50165c66c5f] [error="Error 1050 (42S01): Table 'ddl_reentrant.t4' already exists"] run task successfully ------------------------------------------ test ddl alter table ddl_reentrant.t3 exchange partition p0 with table ddl_reentrant.t4, is_reentrant: true restored_sql: ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4` check_ts_forward 9d8cdff7-a113-4d42-a709-c50165c66c5f owner pid: 6296 owner id 6fa32d8a-a720-4a9f-b696-0a2dde70aed5 process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v '6fa32d8a-a720-4a9f-b696-0a2dde70aed5' /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac47d2 run task successfully process status: S nonempty select id, val from test.availability1 where id=2 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:34:23 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/8d543c8a-4fc3-4885-982c-957387d0b84d {UpstreamID:7365755495359373414 Namespace:default ID:8d543c8a-4fc3-4885-982c-957387d0b84d SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:33:28.96544062 +0800 CST StartTs:449570041713917957 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00360ecf0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570041740132362} {CheckpointTs:449570042434813988 MinTableBarrierTs:449570055843479555 AdminJobType:noop} span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449570055843479554, checkpointTs: 449570042434813988, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570055843479554, checkpointTs: 449570042434813988, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570055843479553, checkpointTs: 449570042434813988, state: Replicating span: {table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}, resolvedTs: 449570055843479555, checkpointTs: 449570042434813988, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449570055843479555, checkpointTs: 449570042434813988, state: Replicating span: {table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}, resolvedTs: 449570042434813988, checkpointTs: 449570042434813988, state: Prepared span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449570055843479554, checkpointTs: 449570042434813988, state: Replicating span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449570055843479555, checkpointTs: 449570042434813988, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d911c9c1-398b-46fe-83b5-bf714000670b {"id":"d911c9c1-398b-46fe-83b5-bf714000670b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973661} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c643f1ede d911c9c1-398b-46fe-83b5-bf714000670b /tidb/cdc/default/default/changefeed/info/8d543c8a-4fc3-4885-982c-957387d0b84d {"upstream-id":7365755495359373414,"namespace":"default","changefeed-id":"8d543c8a-4fc3-4885-982c-957387d0b84d","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:33:28.96544062+08:00","start-ts":449570041713917957,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570041740132362} /tidb/cdc/default/default/changefeed/status/8d543c8a-4fc3-4885-982c-957387d0b84d {"checkpoint-ts":449570042434813988,"min-table-barrier-ts":449570055843479555,"admin-job-type":0} /tidb/cdc/default/default/task/position/d911c9c1-398b-46fe-83b5-bf714000670b/8d543c8a-4fc3-4885-982c-957387d0b84d {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755495359373414 {"id":7365755495359373414,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/8d543c8a-4fc3-4885-982c-957387d0b84d {UpstreamID:7365755495359373414 Namespace:default ID:8d543c8a-4fc3-4885-982c-957387d0b84d SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:33:28.96544062 +0800 CST StartTs:449570041713917957 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00360ecf0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570041740132362} {CheckpointTs:449570042434813988 MinTableBarrierTs:449570055843479555 AdminJobType:noop} span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449570055843479554, checkpointTs: 449570042434813988, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570055843479554, checkpointTs: 449570042434813988, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570055843479553, checkpointTs: 449570042434813988, state: Replicating span: {table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}, resolvedTs: 449570055843479555, checkpointTs: 449570042434813988, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449570055843479555, checkpointTs: 449570042434813988, state: Replicating span: {table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}, resolvedTs: 449570042434813988, checkpointTs: 449570042434813988, state: Prepared span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449570055843479554, checkpointTs: 449570042434813988, state: Replicating span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449570055843479555, checkpointTs: 449570042434813988, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d911c9c1-398b-46fe-83b5-bf714000670b {"id":"d911c9c1-398b-46fe-83b5-bf714000670b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973661} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c643f1ede d911c9c1-398b-46fe-83b5-bf714000670b /tidb/cdc/default/default/changefeed/info/8d543c8a-4fc3-4885-982c-957387d0b84d {"upstream-id":7365755495359373414,"namespace":"default","changefeed-id":"8d543c8a-4fc3-4885-982c-957387d0b84d","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:33:28.96544062+08:00","start-ts":449570041713917957,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570041740132362} /tidb/cdc/default/default/changefeed/status/8d543c8a-4fc3-4885-982c-957387d0b84d {"checkpoint-ts":449570042434813988,"min-table-barrier-ts":449570055843479555,"admin-job-type":0} /tidb/cdc/default/default/task/position/d911c9c1-398b-46fe-83b5-bf714000670b/8d543c8a-4fc3-4885-982c-957387d0b84d {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755495359373414 {"id":7365755495359373414,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/8d543c8a-4fc3-4885-982c-957387d0b84d {UpstreamID:7365755495359373414 Namespace:default ID:8d543c8a-4fc3-4885-982c-957387d0b84d SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:33:28.96544062 +0800 CST StartTs:449570041713917957 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00360ecf0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570041740132362} {CheckpointTs:449570042434813988 MinTableBarrierTs:449570055843479555 AdminJobType:noop} span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449570055843479554, checkpointTs: 449570042434813988, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570055843479554, checkpointTs: 449570042434813988, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570055843479553, checkpointTs: 449570042434813988, state: Replicating span: {table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}, resolvedTs: 449570055843479555, checkpointTs: 449570042434813988, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449570055843479555, checkpointTs: 449570042434813988, state: Replicating span: {table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}, resolvedTs: 449570042434813988, checkpointTs: 449570042434813988, state: Prepared span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449570055843479554, checkpointTs: 449570042434813988, state: Replicating span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449570055843479555, checkpointTs: 449570042434813988, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d911c9c1-398b-46fe-83b5-bf714000670b {"id":"d911c9c1-398b-46fe-83b5-bf714000670b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973661} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c643f1ede d911c9c1-398b-46fe-83b5-bf714000670b /tidb/cdc/default/default/changefeed/info/8d543c8a-4fc3-4885-982c-957387d0b84d {"upstream-id":7365755495359373414,"namespace":"default","changefeed-id":"8d543c8a-4fc3-4885-982c-957387d0b84d","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:33:28.96544062+08:00","start-ts":449570041713917957,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570041740132362} /tidb/cdc/default/default/changefeed/status/8d543c8a-4fc3-4885-982c-957387d0b84d {"checkpoint-ts":449570042434813988,"min-table-barrier-ts":449570055843479555,"admin-job-type":0} /tidb/cdc/default/default/task/position/d911c9c1-398b-46fe-83b5-bf714000670b/8d543c8a-4fc3-4885-982c-957387d0b84d {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365755495359373414 {"id":7365755495359373414,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release changefeed is working normally rts: 449570055776895000->449570056301183000 checkpoint: 449570055776895000->449570056301183000 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/06 13:34:23.883 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4`"] [duration=140.010701ms] [namespace=default] [changefeed=9d8cdff7-a113-4d42-a709-c50165c66c5f] run task successfully + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 11988 23.1 0.5 4427368 2194308 ? Sl 13:33 0:06 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv1 jenkins 11989 22.8 0.5 4437608 2220448 ? Sl 13:33 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv2 jenkins 11990 31.6 0.5 4466788 2261816 ? Sl 13:33 0:08 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv3 jenkins 11992 30.3 0.5 4458600 2252368 ? Sl 13:33 0:07 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 6081 0 --:--:-- --:--:-- --:--:-- 6230 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 13:34:21.683","puller_resolved_ts":"2024-05-06 13:34:17.833","last_synced_ts":"2024-05-06 13:34:17.884","now_ts":"2024-05-06 13:34:24.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '13:34:21.683","puller_resolved_ts":"2024-05-06' '13:34:17.833","last_synced_ts":"2024-05-06' '13:34:17.884","now_ts":"2024-05-06' '13:34:24.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '13:34:21.683","puller_resolved_ts":"2024-05-06' '13:34:17.833","last_synced_ts":"2024-05-06' '13:34:17.884","now_ts":"2024-05-06' '13:34:24.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc already exists) [Pipeline] // cache resume changefeed 9d8cdff7-a113-4d42-a709-c50165c66c5f from 449570055895121933 check_ts_forward 9d8cdff7-a113-4d42-a709-c50165c66c5f [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } nonempty select id, val from test.availability1 where id=2 and val=22 run task successfully wait process cdc.test exit for 1-th time... [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit sql_check run sql_check, 127.0.0.1 run task successfully run test case test_stop_processor [Mon May 6 13:34:27 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.65106512.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_stop_processor.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_stop_processor --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff failed 5-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_auto_stop/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_ts_forward 9d8cdff7-a113-4d42-a709-c50165c66c5f start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... check diff failed 6-th time, retry later Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:34:30 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001a1cea0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570057607446533 MinTableBarrierTs:449570057607446535 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570057607446535, checkpointTs: 449570057607446535, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570057607446533, checkpointTs: 449570057607446533, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570057607446535, checkpointTs: 449570057607446535, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a2a53531-c6fa-4955-8fda-24b7cfed2e13 {"id":"a2a53531-c6fa-4955-8fda-24b7cfed2e13","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973667} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4841 a2a53531-c6fa-4955-8fda-24b7cfed2e13 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570057607446533,"min-table-barrier-ts":449570057607446535,"admin-job-type":0} /tidb/cdc/default/default/task/position/a2a53531-c6fa-4955-8fda-24b7cfed2e13/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001a1cea0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570057607446533 MinTableBarrierTs:449570057607446535 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570057607446535, checkpointTs: 449570057607446535, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570057607446533, checkpointTs: 449570057607446533, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570057607446535, checkpointTs: 449570057607446535, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a2a53531-c6fa-4955-8fda-24b7cfed2e13 {"id":"a2a53531-c6fa-4955-8fda-24b7cfed2e13","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973667} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4841 a2a53531-c6fa-4955-8fda-24b7cfed2e13 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570057607446533,"min-table-barrier-ts":449570057607446535,"admin-job-type":0} /tidb/cdc/default/default/task/position/a2a53531-c6fa-4955-8fda-24b7cfed2e13/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/76bfc1da-dc68-45bf-ac43-3f92d952b920 {UpstreamID:7365754766099386827 Namespace:default ID:76bfc1da-dc68-45bf-ac43-3f92d952b920 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:30:40.10650401 +0800 CST StartTs:449569996095094785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001a1cea0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449569997471350791} {CheckpointTs:449570057607446533 MinTableBarrierTs:449570057607446535 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449570057607446535, checkpointTs: 449570057607446535, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570057607446533, checkpointTs: 449570057607446533, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570057607446535, checkpointTs: 449570057607446535, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a2a53531-c6fa-4955-8fda-24b7cfed2e13 {"id":"a2a53531-c6fa-4955-8fda-24b7cfed2e13","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973667} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4c61ac4841 a2a53531-c6fa-4955-8fda-24b7cfed2e13 /tidb/cdc/default/default/changefeed/info/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"upstream-id":7365754766099386827,"namespace":"default","changefeed-id":"76bfc1da-dc68-45bf-ac43-3f92d952b920","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:30:40.10650401+08:00","start-ts":449569996095094785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449569997471350791} /tidb/cdc/default/default/changefeed/status/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":449570057607446533,"min-table-barrier-ts":449570057607446535,"admin-job-type":0} /tidb/cdc/default/default/task/position/a2a53531-c6fa-4955-8fda-24b7cfed2e13/76bfc1da-dc68-45bf-ac43-3f92d952b920 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365754766099386827 {"id":7365754766099386827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3195e80c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d, pid:36118, start at 2024-05-06 13:34:29.918679842 +0800 CST m=+5.138197415 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:29.928 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:29.891 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:29.891 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3195e80c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d, pid:36118, start at 2024-05-06 13:34:29.918679842 +0800 CST m=+5.138197415 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:29.928 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:29.891 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:29.891 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3195e9880003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d, pid:36201, start at 2024-05-06 13:34:29.987299017 +0800 CST m=+5.158041251 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:29.994 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:29.986 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:29.986 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } changefeed is working normally rts: 449570057612165100->449570057874309100 checkpoint: 449570057612165100->449570057874309100 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/06 13:34:27.731 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4`"] [duration=139.585145ms] [namespace=default] [changefeed=9d8cdff7-a113-4d42-a709-c50165c66c5f] run task successfully "is-owner": true, run task successfully check diff failed 7-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3195fdc40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-0cr08-7wrc3-1z3gv, pid:10296, start at 2024-05-06 13:34:31.32906885 +0800 CST m=+5.155807788 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:31.335 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:31.331 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:31.331 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3195fdc40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-0cr08-7wrc3-1z3gv, pid:10296, start at 2024-05-06 13:34:31.32906885 +0800 CST m=+5.155807788 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:31.335 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:31.331 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:31.331 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319600d80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-0cr08-7wrc3-1z3gv, pid:10383, start at 2024-05-06 13:34:31.480894485 +0800 CST m=+5.232722744 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:31.487 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:31.478 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:31.478 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_remove_table_error/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/owner_remove_table_error/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls_with_user_auth Starting TLS PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:34:33 CST 2024] <<<<<< run test case ddl_reentrant success! >>>>>> owner pid: 6515 owner id a2a53531-c6fa-4955-8fda-24b7cfed2e13 Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 8-th time, retry later [Mon May 6 13:34:34 CST 2024] <<<<<< START cdc server in owner_remove_table_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/OwnerRemoveTableError=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_remove_table_error.1168211684.out server --log-file /tmp/tidb_cdc_test/owner_remove_table_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_remove_table_error/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 changefeed id: 76bfc1da-dc68-45bf-ac43-3f92d952b920 curl -s -d "cf-id=76bfc1da-dc68-45bf-ac43-3f92d952b920&admin-job=1" http://127.0.0.1:8300/capture/owner/admin | grep true "status": true, run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 89 100 35 100 54 451 696 --:--:-- --:--:-- --:--:-- 701 { "status": true, "message": "" }nonempty select id, val from test.availability1 where id=4 and val=4 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 9-th time, retry later nonempty select id, val from test.availability1 where id=4 and val=4 run task successfully test_stop_processor pass ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:34:37 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a9d8e9c-fb4d-4831-ad71-27fde0b20e5b {"id":"3a9d8e9c-fb4d-4831-ad71-27fde0b20e5b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973674} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c655795c6 3a9d8e9c-fb4d-4831-ad71-27fde0b20e5b /tidb/cdc/default/default/upstream/7365755797386165040 {"id":7365755797386165040,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a9d8e9c-fb4d-4831-ad71-27fde0b20e5b {"id":"3a9d8e9c-fb4d-4831-ad71-27fde0b20e5b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973674} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c655795c6 3a9d8e9c-fb4d-4831-ad71-27fde0b20e5b /tidb/cdc/default/default/upstream/7365755797386165040 {"id":7365755797386165040,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a9d8e9c-fb4d-4831-ad71-27fde0b20e5b {"id":"3a9d8e9c-fb4d-4831-ad71-27fde0b20e5b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973674} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c655795c6 3a9d8e9c-fb4d-4831-ad71-27fde0b20e5b /tidb/cdc/default/default/upstream/7365755797386165040 {"id":7365755797386165040,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff failed 10-th time, retry later wait process cdc.test exit for 2-th time... Starting TLS TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:34:38 CST 2024] <<<<<< run test case availability success! >>>>>> table owner_remove_table_error.finished_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting TLS TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table owner_remove_table_error.finished_mark not exists for 2-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319688700005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:24845, start at 2024-05-06 13:34:40.160435838 +0800 CST m=+5.167972342 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:40.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:40.156 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:40.156 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319688700005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:24845, start at 2024-05-06 13:34:40.160435838 +0800 CST m=+5.167972342 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:40.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:40.156 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:40.156 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3196891c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:24928, start at 2024-05-06 13:34:40.221946896 +0800 CST m=+5.177963011 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:40.231 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:40.199 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:40.199 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:34:41 CST 2024] <<<<<< run test case sink_hang success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table owner_remove_table_error.finished_mark not exists for 3-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_cdc_cluster/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.26184.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3196b1740015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54, pid:10046, start at 2024-05-06 13:34:42.827359804 +0800 CST m=+5.168465319 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:42.836 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:42.832 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:42.832 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3196b1740015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54, pid:10046, start at 2024-05-06 13:34:42.827359804 +0800 CST m=+5.168465319 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:42.836 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:42.832 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:42.832 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3196b4780014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-36hxm-r68df-z4t54, pid:10130, start at 2024-05-06 13:34:42.995062644 +0800 CST m=+5.268359302 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:43.004 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:42.974 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:42.974 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table owner_remove_table_error.finished_mark exists check diff successfully wait process cdc.test exit for 1-th time... + set +x + tso='449570061315473409 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570061315473409 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x try a VALID cdc server command [Mon May 6 13:34:45 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.2622326225.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check_redo_resolved_ts 8d543c8a-4fc3-4885-982c-957387d0b84d 449570061702397958 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/8d543c8a-4fc3-4885-982c-957387d0b84d/meta global resolved ts 449570056131575828 not forward to 449570061702397958 run task failed 1-th time, retry later wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3196ccd4000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d, pid:37863, start at 2024-05-06 13:34:44.547210238 +0800 CST m=+5.064127772 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:44.553 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:44.533 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:44.533 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Mon May 6 13:34:45 CST 2024] <<<<<< START cdc server in http_api_tls_with_user_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls_with_user_auth.3792337925.out server --log-file /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_http_api_tls_with_user_auth_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_data_http_api_tls_with_user_auth_tls1 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls_with_user_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:34:46 CST 2024] <<<<<< run test case owner_remove_table_error success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/multi_cdc_cluster Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ***************** properties ***************** "mysql.port"="4000" "updateproportion"="0" "mysql.db"="changefeed_auto_stop_1" "threadcount"="4" "workload"="core" "operationcount"="0" "requestdistribution"="uniform" "scanproportion"="0" "dotransactions"="false" "readallfields"="true" "mysql.host"="127.0.0.1" "recordcount"="20" "readproportion"="0" "insertproportion"="0" "mysql.user"="root" ********************************************** Run finished, takes 8.620393ms INSERT - Takes(s): 0.0, Count: 20, OPS: 4054.0, Avg(us): 1620, Min(us): 1009, Max(us): 3597, 95th(us): 4000, 99th(us): 4000 check_redo_resolved_ts 8d543c8a-4fc3-4885-982c-957387d0b84d 449570061702397958 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/8d543c8a-4fc3-4885-982c-957387d0b84d/meta global resolved ts 449570056131575828 not forward to 449570061702397958 run task failed 2-th time, retry later Verifying downstream PD is started... ***************** properties ***************** "mysql.user"="root" "operationcount"="0" "mysql.port"="4000" "dotransactions"="false" "mysql.db"="changefeed_auto_stop_2" "threadcount"="4" "readallfields"="true" "scanproportion"="0" "recordcount"="20" "workload"="core" "readproportion"="0" "insertproportion"="0" "mysql.host"="127.0.0.1" "updateproportion"="0" "requestdistribution"="uniform" ********************************************** Run finished, takes 8.688482ms INSERT - Takes(s): 0.0, Count: 19, OPS: 3978.7, Avg(us): 1498, Min(us): 903, Max(us): 3758, 95th(us): 4000, 99th(us): 4000 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/http_proxies + CDC_BINARY=cdc.test + SINK_TYPE=mysql + TEST_HOST_LIST=(UP_TIDB_HOST UP_PD_HOST_{1..3} UP_TIKV_HOST_{1..3}) + lan_addrs=($(hostname -I)) ++ hostname -I + lan_addr=10.233.127.82 + export UP_TIDB_HOST=10.233.127.82 UP_PD_HOST_1=10.233.127.82 UP_PD_HOST_2=10.233.127.82 UP_PD_HOST_3=10.233.127.82 UP_TIKV_HOST_1=10.233.127.82 UP_TIKV_HOST_2=10.233.127.82 UP_TIKV_HOST_3=10.233.127.82 + UP_TIDB_HOST=10.233.127.82 + UP_PD_HOST_1=10.233.127.82 + UP_PD_HOST_2=10.233.127.82 + UP_PD_HOST_3=10.233.127.82 + UP_TIKV_HOST_1=10.233.127.82 + UP_TIKV_HOST_2=10.233.127.82 + UP_TIKV_HOST_3=10.233.127.82 + proxy_pid= ++ shuf -i 10081-20081 -n1 + proxy_port=11934 + trap 'stop_tidb_cluster && stop_proxy' EXIT + '[' mysql == mysql ']' + prepare + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIDB_HOST + echo 'UP_TIDB_HOST 10.233.127.82' UP_TIDB_HOST 10.233.127.82 + case $(printenv $host) in ++ printenv UP_TIDB_HOST + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_1 + echo 'UP_PD_HOST_1 10.233.127.82' UP_PD_HOST_1 10.233.127.82 + case $(printenv $host) in ++ printenv UP_PD_HOST_1 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_2 + echo 'UP_PD_HOST_2 10.233.127.82' UP_PD_HOST_2 10.233.127.82 + case $(printenv $host) in ++ printenv UP_PD_HOST_2 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_3 + echo 'UP_PD_HOST_3 10.233.127.82' UP_PD_HOST_3 10.233.127.82 + case $(printenv $host) in ++ printenv UP_PD_HOST_3 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_1 + echo 'UP_TIKV_HOST_1 10.233.127.82' UP_TIKV_HOST_1 10.233.127.82 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_1 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_2 + echo 'UP_TIKV_HOST_2 10.233.127.82' UP_TIKV_HOST_2 10.233.127.82 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_2 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_3 + echo 'UP_TIKV_HOST_3 10.233.127.82' UP_TIKV_HOST_3 10.233.127.82 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_3 + rm -rf /tmp/tidb_cdc_test/http_proxies + mkdir -p /tmp/tidb_cdc_test/http_proxies + stop_tidb_cluster + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:34:48 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d527d71-721e-49a7-b24f-a6812f264533 {"id":"2d527d71-721e-49a7-b24f-a6812f264533","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973685} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c657a0cdd 2d527d71-721e-49a7-b24f-a6812f264533 /tidb/cdc/default/default/upstream/7365755833128253436 {"id":7365755833128253436,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d527d71-721e-49a7-b24f-a6812f264533 {"id":"2d527d71-721e-49a7-b24f-a6812f264533","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973685} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c657a0cdd 2d527d71-721e-49a7-b24f-a6812f264533 /tidb/cdc/default/default/upstream/7365755833128253436 {"id":7365755833128253436,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d527d71-721e-49a7-b24f-a6812f264533 {"id":"2d527d71-721e-49a7-b24f-a6812f264533","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973685} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c657a0cdd 2d527d71-721e-49a7-b24f-a6812f264533 /tidb/cdc/default/default/upstream/7365755833128253436 {"id":7365755833128253436,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:34:48 GMT < Content-Length: 1209 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbe49b3d-e01a-4ebd-96a2-849a38054e11 {"id":"dbe49b3d-e01a-4ebd-96a2-849a38054e11","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973686} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c6586d468 dbe49b3d-e01a-4ebd-96a2-849a38054e11 /tidb/cdc/default/default/upstream/7365755854047880908 {"id":7365755854047880908,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbe49b3d-e01a-4ebd-96a2-849a38054e11 {"id":"dbe49b3d-e01a-4ebd-96a2-849a38054e11","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973686} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c6586d468 dbe49b3d-e01a-4ebd-96a2-849a38054e11 /tidb/cdc/default/default/upstream/7365755854047880908 {"id":7365755854047880908,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbe49b3d-e01a-4ebd-96a2-849a38054e11 {"id":"dbe49b3d-e01a-4ebd-96a2-849a38054e11","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973686} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c6586d468 dbe49b3d-e01a-4ebd-96a2-849a38054e11 /tidb/cdc/default/default/upstream/7365755854047880908 {"id":7365755854047880908,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ***************** properties ***************** "readallfields"="true" "mysql.host"="127.0.0.1" "mysql.port"="4000" "workload"="core" "threadcount"="4" "scanproportion"="0" "insertproportion"="0" "requestdistribution"="uniform" "recordcount"="20" "mysql.db"="changefeed_auto_stop_3" "operationcount"="0" "dotransactions"="false" "readproportion"="0" "updateproportion"="0" "mysql.user"="root" ********************************************** Run finished, takes 9.159449ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3764.1, Avg(us): 1721, Min(us): 985, Max(us): 4014, 95th(us): 4000, 99th(us): 5000 + start_tidb_cluster --workdir /tmp/tidb_cdc_test/http_proxies The 1 times to try to start tidb cluster... [Mon May 6 13:34:50 CST 2024] <<<<<< START cdc server in http_api_tls_with_user_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls_with_user_auth.3798437986.out server --log-file /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_http_api_tls_with_user_auth_tls2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_data_http_api_tls_with_user_auth_tls2 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls_with_user_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8301 --pd https://127.0.0.1:2579 + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check_redo_resolved_ts 8d543c8a-4fc3-4885-982c-957387d0b84d 449570061702397958 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/8d543c8a-4fc3-4885-982c-957387d0b84d/meta run task successfully ***************** properties ***************** "insertproportion"="0" "readproportion"="0" "mysql.user"="root" "mysql.db"="changefeed_auto_stop_4" "requestdistribution"="uniform" "mysql.port"="4000" "workload"="core" "dotransactions"="false" "updateproportion"="0" "recordcount"="20" "operationcount"="0" "readallfields"="true" "scanproportion"="0" "threadcount"="4" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 8.541908ms INSERT - Takes(s): 0.0, Count: 20, OPS: 4262.6, Avg(us): 1631, Min(us): 947, Max(us): 3790, 95th(us): 4000, 99th(us): 4000 [Mon May 6 13:34:51 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1153911541.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... wait process cdc.test exit for 1-th time... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [2024/05/06 13:34:53.681 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 13:34:53.683 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2024/05/06 13:34:53.683 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=:6060] [2024/05/06 13:34:53.683 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 13:34:53.683 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449570056184266761] [resolvedTs=449570063196618758] [2024/05/06 13:34:53.685 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 13:34:53.685 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 13:34:53.686 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2024/05/06 13:34:53.687 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=4.580157ms] [2024/05/06 13:34:53.690 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 13:34:53.693 +08:00] [INFO] [mysql.go:197] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 13:34:53.694 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:34:53.694 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:34:53.694 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:34:53.694 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:34:53.694 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:34:53.694 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:34:53.694 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:34:53.694 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:34:53.694 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:34:53.694 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:34:53.694 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:34:53.694 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:34:53.694 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:34:53.694 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:34:53.694 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:34:53.694 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/06 13:34:53.694 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 13:34:53.694 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 13:34:53.694 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 13:34:53.694 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 13:34:53.694 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 13:34:53.694 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 13:34:53.694 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 13:34:53.694 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 13:34:53.694 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 13:34:53.694 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 13:34:53.694 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 13:34:53.694 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 13:34:53.694 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 13:34:53.694 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 13:34:53.694 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 13:34:53.694 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 13:34:53.694 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 13:34:53.694 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 13:34:53.694 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2024/05/06 13:34:53.698 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 13:34:53.701 +08:00] [INFO] [mysql_ddl_sink.go:107] ["MySQL DDL sink is created"] [namespace=] [changefeed=] \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/http_proxies Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] // timeout + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:34:54 GMT < Content-Length: 1659 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbe49b3d-e01a-4ebd-96a2-849a38054e11 {"id":"dbe49b3d-e01a-4ebd-96a2-849a38054e11","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973686} /tidb/cdc/default/__cdc_meta__/capture/ecb172c8-6b3a-4bb0-baa2-4f75ad993107 {"id":"ecb172c8-6b3a-4bb0-baa2-4f75ad993107","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973691} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c6586d468 dbe49b3d-e01a-4ebd-96a2-849a38054e11 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c6586d4aa ecb172c8-6b3a-4bb0-baa2-4f75ad993107 /tidb/cdc/default/default/upstream/7365755854047880908 {"id":7365755854047880908,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbe49b3d-e01a-4ebd-96a2-849a38054e11 {"id":"dbe49b3d-e01a-4ebd-96a2-849a38054e11","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973686} /tidb/cdc/default/__cdc_meta__/capture/ecb172c8-6b3a-4bb0-baa2-4f75ad993107 {"id":"ecb172c8-6b3a-4bb0-baa2-4f75ad993107","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973691} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c6586d468 dbe49b3d-e01a-4ebd-96a2-849a38054e11 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c6586d4aa ecb172c8-6b3a-4bb0-baa2-4f75ad993107 /tidb/cdc/default/default/upstream/7365755854047880908 {"id":7365755854047880908,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbe49b3d-e01a-4ebd-96a2-849a38054e11 {"id":"dbe49b3d-e01a-4ebd-96a2-849a38054e11","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973686} /tidb/cdc/default/__cdc_meta__/capture/ecb172c8-6b3a-4bb0-baa2-4f75ad993107 {"id":"ecb172c8-6b3a-4bb0-baa2-4f75ad993107","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973691} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c6586d468 dbe49b3d-e01a-4ebd-96a2-849a38054e11 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c6586d4aa ecb172c8-6b3a-4bb0-baa2-4f75ad993107 /tidb/cdc/default/default/upstream/7365755854047880908 {"id":7365755854047880908,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x [2024/05/06 13:34:54.353 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=669.693456ms] [2024/05/06 13:34:54.353 +08:00] [ERROR] [redo.go:268] ["ignore unsupported DDL"] [ddl="{\"StartTs\":449570056184266755,\"CommitTs\":449570056184266761,\"Query\":\"ALTER TABLE `consistent_replicate_storage_file`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_file`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_storage_file\",\"Table\":\"t1\",\"TableID\":110,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:34:54 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a3c07bd-6b38-46b8-977b-aa59f358895b {"id":"4a3c07bd-6b38-46b8-977b-aa59f358895b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973692} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c65806751 4a3c07bd-6b38-46b8-977b-aa59f358895b /tidb/cdc/default/default/upstream/7365755836589514652 {"id":7365755836589514652,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a3c07bd-6b38-46b8-977b-aa59f358895b {"id":"4a3c07bd-6b38-46b8-977b-aa59f358895b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973692} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c65806751 4a3c07bd-6b38-46b8-977b-aa59f358895b /tidb/cdc/default/default/upstream/7365755836589514652 {"id":7365755836589514652,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a3c07bd-6b38-46b8-977b-aa59f358895b {"id":"4a3c07bd-6b38-46b8-977b-aa59f358895b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973692} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c65806751 4a3c07bd-6b38-46b8-977b-aa59f358895b /tidb/cdc/default/default/upstream/7365755836589514652 {"id":7365755836589514652,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 13:34:55 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedError=1*return(true);github.com/pingcap/tiflow/cdc/processor/ProcessorUpdatePositionDelaying=sleep(1000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1159611598.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // stage [Pipeline] } [2024/05/06 13:34:55.422 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}] [checkpointTs=449570063196618758] [2024/05/06 13:34:55.422 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}] [checkpointTs=449570063196618758] [2024/05/06 13:34:55.522 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}] [checkpointTs=449570063196618758] [2024/05/06 13:34:55.522 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}] [checkpointTs=449570063196618758] Verifying downstream PD is started... pass test: check health pass test: get status [2024/05/06 13:34:56.631 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:120,start_key:7480000000000000ff785f720000000000fa,end_key:7480000000000000ff785f730000000000fa}] [checkpointTs=449570063196618758] [2024/05/06 13:34:56.631 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:120,start_key:7480000000000000ff785f720000000000fa,end_key:7480000000000000ff785f730000000000fa}] [checkpointTs=449570063196618758] [2024/05/06 13:34:56.731 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=5008] [appliedDDLCount=0] [currentCheckpoint=449570063196618758] [2024/05/06 13:34:56.731 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 13:34:56.731 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 13:34:56.731 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 13:34:56.731 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 13:34:56.731 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 13:34:56.731 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 13:34:56.731 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 13:34:56.731 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 13:34:56.731 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/06 13:34:56.731 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/06 13:34:56.731 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/06 13:34:56.731 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 13:34:56.731 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/06 13:34:56.731 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 13:34:56.731 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/06 13:34:56.731 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] Apply redo log successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully [Mon May 6 13:34:57 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> count(*) 5000 pass test: create changefeed check_changefeed_state https://127.0.0.1:2579 changefeed-test1 normal null /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test1 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test1 -s + info='{ "upstream_id": 7365755854047880908, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 449570064703946772, "checkpoint_time": "2024-05-06 13:34:56.533", "error": null }' + echo '{ "upstream_id": 7365755854047880908, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 449570064703946772, "checkpoint_time": "2024-05-06 13:34:56.533", "error": null }' { "upstream_id": 7365755854047880908, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 449570064703946772, "checkpoint_time": "2024-05-06 13:34:56.533", "error": null } ++ echo '{' '"upstream_id":' 7365755854047880908, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 449570064703946772, '"checkpoint_time":' '"2024-05-06' '13:34:56.533",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365755854047880908, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 449570064703946772, '"checkpoint_time":' '"2024-05-06' '13:34:56.533",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test2 normal null /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test2 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test2 -s + info='{ "upstream_id": 7365755854047880908, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 449570064769482759, "checkpoint_time": "2024-05-06 13:34:56.783", "error": null }' + echo '{ "upstream_id": 7365755854047880908, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 449570064769482759, "checkpoint_time": "2024-05-06 13:34:56.783", "error": null }' { "upstream_id": 7365755854047880908, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 449570064769482759, "checkpoint_time": "2024-05-06 13:34:56.783", "error": null } ++ echo '{' '"upstream_id":' 7365755854047880908, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 449570064769482759, '"checkpoint_time":' '"2024-05-06' '13:34:56.783",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365755854047880908, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 449570064769482759, '"checkpoint_time":' '"2024-05-06' '13:34:56.783",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test3 normal null /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test3 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test3 -s + info='{ "upstream_id": 7365755854047880908, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 449570064808804372, "checkpoint_time": "2024-05-06 13:34:56.933", "error": null }' + echo '{ "upstream_id": 7365755854047880908, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 449570064808804372, "checkpoint_time": "2024-05-06 13:34:56.933", "error": null }' { "upstream_id": 7365755854047880908, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 449570064808804372, "checkpoint_time": "2024-05-06 13:34:56.933", "error": null } ++ echo '{' '"upstream_id":' 7365755854047880908, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 449570064808804372, '"checkpoint_time":' '"2024-05-06' '13:34:56.933",' '"error":' null '}' ++ jq -r .state \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365755854047880908, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 449570064808804372, '"checkpoint_time":' '"2024-05-06' '13:34:56.933",' '"error":' null '}' + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test4 normal null /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test4 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test4 -s [Pipeline] // dir [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:34:58 GMT < Content-Length: 1259 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a3c07bd-6b38-46b8-977b-aa59f358895b {"id":"4a3c07bd-6b38-46b8-977b-aa59f358895b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973692} /tidb/cdc/default/__cdc_meta__/capture/52916971-959c-41a9-b7c2-2e6b80d6062a {"id":"52916971-959c-41a9-b7c2-2e6b80d6062a","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973695} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c65806751 4a3c07bd-6b38-46b8-977b-aa59f358895b /tidb/cdc/default/__cdc_meta__/owner/22318f4c65806785 52916971-959c-41a9-b7c2-2e6b80d6062a /tidb/cdc/default/default/upstream/7365755836589514652 {"id":7365755836589514652,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a3c07bd-6b38-46b8-977b-aa59f358895b {"id":"4a3c07bd-6b38-46b8-977b-aa59f358895b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973692} /tidb/cdc/default/__cdc_meta__/capture/52916971-959c-41a9-b7c2-2e6b80d6062a {"id":"52916971-959c-41a9-b7c2-2e6b80d6062a","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973695} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c65806751 4a3c07bd-6b38-46b8-977b-aa59f358895b /tidb/cdc/default/__cdc_meta__/owner/22318f4c65806785 52916971-959c-41a9-b7c2-2e6b80d6062a /tidb/cdc/default/default/upstream/7365755836589514652 {"id":7365755836589514652,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a3c07bd-6b38-46b8-977b-aa59f358895b {"id":"4a3c07bd-6b38-46b8-977b-aa59f358895b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973692} /tidb/cdc/default/__cdc_meta__/capture/52916971-959c-41a9-b7c2-2e6b80d6062a {"id":"52916971-959c-41a9-b7c2-2e6b80d6062a","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973695} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c65806751 4a3c07bd-6b38-46b8-977b-aa59f358895b /tidb/cdc/default/__cdc_meta__/owner/22318f4c65806785 52916971-959c-41a9-b7c2-2e6b80d6062a /tidb/cdc/default/default/upstream/7365755836589514652 {"id":7365755836589514652,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 5796378f-76e8-4101-bfaf-2346bf597410 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=5796378f-76e8-4101-bfaf-2346bf597410 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 5796378f-76e8-4101-bfaf-2346bf597410 -s + info='{ "upstream_id": 7365755836589514652, "namespace": "default", "id": "5796378f-76e8-4101-bfaf-2346bf597410", "state": "normal", "checkpoint_tso": 449570061951172609, "checkpoint_time": "2024-05-06 13:34:46.032", "error": null }' + echo '{ "upstream_id": 7365755836589514652, "namespace": "default", "id": "5796378f-76e8-4101-bfaf-2346bf597410", "state": "normal", "checkpoint_tso": 449570061951172609, "checkpoint_time": "2024-05-06 13:34:46.032", "error": null }' { "upstream_id": 7365755836589514652, "namespace": "default", "id": "5796378f-76e8-4101-bfaf-2346bf597410", "state": "normal", "checkpoint_tso": 449570061951172609, "checkpoint_time": "2024-05-06 13:34:46.032", "error": null } ++ echo '{' '"upstream_id":' 7365755836589514652, '"namespace":' '"default",' '"id":' '"5796378f-76e8-4101-bfaf-2346bf597410",' '"state":' '"normal",' '"checkpoint_tso":' 449570061951172609, '"checkpoint_time":' '"2024-05-06' '13:34:46.032",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365755836589514652, '"namespace":' '"default",' '"id":' '"5796378f-76e8-4101-bfaf-2346bf597410",' '"state":' '"normal",' '"checkpoint_tso":' 449570061951172609, '"checkpoint_time":' '"2024-05-06' '13:34:46.032",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table changefeed_auto_stop_1.usertable not exists for 1-th check, retry later [Pipeline] // withCredentials [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // timeout + info='{ "upstream_id": 7365755854047880908, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 449570064861233169, "checkpoint_time": "2024-05-06 13:34:57.133", "error": null }' + echo '{ "upstream_id": 7365755854047880908, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 449570064861233169, "checkpoint_time": "2024-05-06 13:34:57.133", "error": null }' { "upstream_id": 7365755854047880908, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 449570064861233169, "checkpoint_time": "2024-05-06 13:34:57.133", "error": null } ++ echo '{' '"upstream_id":' 7365755854047880908, '"namespace":' '"default",' '"id":' '"changefeed-test4",' '"state":' '"normal",' '"checkpoint_tso":' 449570064861233169, '"checkpoint_time":' '"2024-05-06' '13:34:57.133",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365755854047880908, '"namespace":' '"default",' '"id":' '"changefeed-test4",' '"state":' '"normal",' '"checkpoint_tso":' 449570064861233169, '"checkpoint_time":' '"2024-05-06' '13:34:57.133",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully [Pipeline] } Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table changefeed_auto_stop_1.usertable exists table changefeed_auto_stop_2.usertable not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3197959c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-855t7-fbskf-4lwp5, pid:11957, start at 2024-05-06 13:34:57.425481241 +0800 CST m=+5.227010834 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:57.435 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:57.432 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:57.432 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3197959c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-855t7-fbskf-4lwp5, pid:11957, start at 2024-05-06 13:34:57.425481241 +0800 CST m=+5.227010834 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:57.435 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:57.432 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:57.432 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319796fc0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-855t7-fbskf-4lwp5, pid:12045, start at 2024-05-06 13:34:57.505967531 +0800 CST m=+5.249374308 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:36:57.512 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:34:57.471 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:24:57.471 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_cdc_cluster/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.127.82' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.13349.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.127.82' (111) table test.simple1 exists running list_changefeed pass test: list changefeed table changefeed_auto_stop_2.usertable exists table changefeed_auto_stop_3.usertable not exists for 1-th check, retry later running get_changefeed pass test: get changefeed + set +x + tso='449570066355453953 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570066355453953 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.127.82' (111) table changefeed_auto_stop_3.usertable exists table changefeed_auto_stop_4.usertable not exists for 1-th check, retry later running pause_changefeed pass test: pause changefeed [Mon May 6 13:35:05 CST 2024] <<<<<< START cdc server in multi_cdc_cluster case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.1340313405.out server --log-file /tmp/tidb_cdc_test/multi_cdc_cluster/cdcmult_cdc.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_cdc_cluster/cdc_datamult_cdc.server1 --cluster-id test1 --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 running update_changefeed table changefeed_auto_stop_4.usertable not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319812500017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-wn6pn-dr756-hx56v, pid:7742, start at 2024-05-06 13:35:05.406457162 +0800 CST m=+5.163194920 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:37:05.413 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:35:05.414 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:25:05.414 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319812500017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-wn6pn-dr756-hx56v, pid:7742, start at 2024-05-06 13:35:05.406457162 +0800 CST m=+5.163194920 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:37:05.413 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:35:05.414 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:25:05.414 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319813d00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-wn6pn-dr756-hx56v, pid:7827, start at 2024-05-06 13:35:05.496990492 +0800 CST m=+5.163823969 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:37:05.507 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:35:05.510 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:25:05.510 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_proxies/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_proxies/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/http_proxies/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["10.233.127.82:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/http_proxies/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_proxies/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } pass test: update changefeed =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... running resume_changefeed table changefeed_auto_stop_4.usertable exists pass test: resume changefeed + start_proxy + echo 'dumpling grpc packet to /tmp/tidb_cdc_test/http_proxies/test_proxy.log...' dumpling grpc packet to /tmp/tidb_cdc_test/http_proxies/test_proxy.log... + proxy_pid=9123 + echo 'proxy port: 11934' proxy port: 11934 + echo 'proxy pid: 9123' proxy pid: 9123 + sleep 5 + GO111MODULE=on + WORK_DIR=/tmp/tidb_cdc_test/http_proxies + go run /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies/run-proxy.go --port=11934 go: downloading google.golang.org/grpc v1.62.1 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/pkg/errors v0.9.1 go: downloading golang.org/x/sys v0.19.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f go: downloading github.com/gorilla/websocket v1.5.1 check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:35:08 GMT < Content-Length: 801 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/e2b6fef2-9b66-401b-9cf3-e29103150382 {"id":"e2b6fef2-9b66-401b-9cf3-e29103150382","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973705} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22318f4c65b93ef8 e2b6fef2-9b66-401b-9cf3-e29103150382 /tidb/cdc/test1/default/upstream/7365755902203132484 {"id":7365755902203132484,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/e2b6fef2-9b66-401b-9cf3-e29103150382 {"id":"e2b6fef2-9b66-401b-9cf3-e29103150382","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973705} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22318f4c65b93ef8 e2b6fef2-9b66-401b-9cf3-e29103150382 /tidb/cdc/test1/default/upstream/7365755902203132484 {"id":7365755902203132484,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/e2b6fef2-9b66-401b-9cf3-e29103150382 {"id":"e2b6fef2-9b66-401b-9cf3-e29103150382","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973705} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22318f4c65b93ef8 e2b6fef2-9b66-401b-9cf3-e29103150382 /tidb/cdc/test1/default/upstream/7365755902203132484 {"id":7365755902203132484,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 13:35:08 CST 2024] <<<<<< START cdc server in multi_cdc_cluster case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.1345213454.out server --log-file /tmp/tidb_cdc_test/multi_cdc_cluster/cdcmult_cdc.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_cdc_cluster/cdc_datamult_cdc.server2 --cluster-id test2 --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda running rebalance_table pass test: rebalance table valid ~~~ running cdc Failed to start cdc, the usage tips should be printed 1st test case cdc_server_tips success! try an INVALID cdc server command [Mon May 6 13:35:08 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ true != \n\o ]] + GO_FAILPOINTS= + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.2630926311.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default --pd None start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... running list_processor pass test: list processors check diff successfully wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:35:11 GMT < Content-Length: 801 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/79ba5176-41e8-4ea0-b01c-7366fa13a250 {"id":"79ba5176-41e8-4ea0-b01c-7366fa13a250","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973708} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22318f4c65b93f1f 79ba5176-41e8-4ea0-b01c-7366fa13a250 /tidb/cdc/test2/default/upstream/7365755902203132484 {"id":7365755902203132484,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/79ba5176-41e8-4ea0-b01c-7366fa13a250 {"id":"79ba5176-41e8-4ea0-b01c-7366fa13a250","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973708} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22318f4c65b93f1f 79ba5176-41e8-4ea0-b01c-7366fa13a250 /tidb/cdc/test2/default/upstream/7365755902203132484 {"id":7365755902203132484,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/79ba5176-41e8-4ea0-b01c-7366fa13a250 {"id":"79ba5176-41e8-4ea0-b01c-7366fa13a250","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973708} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22318f4c65b93f1f 79ba5176-41e8-4ea0-b01c-7366fa13a250 /tidb/cdc/test2/default/upstream/7365755902203132484 {"id":7365755902203132484,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.13508.out cli changefeed create --start-ts=449570066355453953 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server http://127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_cdc_cluster/conf/changefeed1.toml wait process cdc.test exit for 2-th time... Create changefeed successfully! ID: a2c2dc4a-68fe-4791-ab3d-1b763897bf81 Info: {"upstream_id":7365755902203132484,"namespace":"default","id":"a2c2dc4a-68fe-4791-ab3d-1b763897bf81","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:35:11.974117052+08:00","start_ts":449570066355453953,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["test.multi_cdc1"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570066355453953,"checkpoint_ts":449570066355453953,"checkpoint_time":"2024-05-06 13:35:02.833"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... running get_processor wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:35:12 CST 2024] <<<<<< run test case changefeed_auto_stop success! >>>>>> [2024/05/06 13:35:11.826 +08:00] [WARN] [diff.go:745] ["rows is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] [cost=1m35.060995902s] [2024/05/06 13:35:11.829 +08:00] [WARN] [diff.go:384] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.13561.out cli changefeed create --start-ts=449570066355453953 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server http://127.0.0.1:8301 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_cdc_cluster/conf/changefeed2.toml Create changefeed successfully! ID: 13e7e7bf-cb71-4ba3-b6e8-873e45b5b74b Info: {"upstream_id":7365755902203132484,"namespace":"default","id":"13e7e7bf-cb71-4ba3-b6e8-873e45b5b74b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:35:13.868963646+08:00","start_ts":449570066355453953,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["test.multi_cdc2"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570066355453953,"checkpoint_ts":449570066355453953,"checkpoint_time":"2024-05-06 13:35:02.833"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + export http_proxy=http://127.0.0.1:11934 + http_proxy=http://127.0.0.1:11934 + export https_proxy=http://127.0.0.1:11934 + https_proxy=http://127.0.0.1:11934 + echo 'try to connect pd cluster via proxy, pd addr: 10.233.127.82:2379' try to connect pd cluster via proxy, pd addr: 10.233.127.82:2379 + ensure 10 curl http://10.233.127.82:2379/ curl http://10.233.127.82:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:11934; Connection refused run task failed 1-th time, retry later pass test: get processor Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x table test.multi_cdc1 exists table test.multi_cdc2 exists check diff failed 1-th time, retry later curl http://10.233.127.82:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:11934; Connection refused run task failed 2-th time, retry later running move_table pass test: move table Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) running set_log_level pass test: set log level check diff successfully wait process cdc.test exit for 1-th time... running remove_changefeed [2024/05/06 13:35:17.826 +08:00] [INFO] [dailytest.go:68] ["test pass!!!"] wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: remove changefeed wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... wait process cdc.test exit for 3-th time... curl http://10.233.127.82:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:11934; Connection refused run task failed 3-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:35:19 CST 2024] <<<<<< run test case multi_cdc_cluster success! >>>>>> running resign_owner pass test: resign owner cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:35:20 CST 2024] <<<<<< run test case cdc success! >>>>>> running get_tso time="2024-05-06T13:35:20+08:00" level=info msg="Listening on 127.0.0.1:11934" time="2024-05-06T13:35:20+08:00" level=info msg="Not intercepting TLS connections" ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: get tso running verify_table pass test: verify table VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319918840012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz, pid:12779, start at 2024-05-06 13:35:22.172808758 +0800 CST m=+5.546416101 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:37:22.180 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:35:22.145 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:25:22.145 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) running create_changefeed_v2 pass test: create changefeed v2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319918840012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz, pid:12779, start at 2024-05-06 13:35:22.172808758 +0800 CST m=+5.546416101 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:37:22.180 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:35:22.145 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:25:22.145 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319919f00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-2dpm3-xs70c-mtvkz, pid:12870, start at 2024-05-06 13:35:22.259041941 +0800 CST m=+5.579622786 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:37:22.265 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:35:22.236 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:25:22.236 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } running delete_changefeed_v2 [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } pass test: remove changefeed v2 curl http://10.233.127.82:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 19 100 19 0 0 14032 0 --:--:-- --:--:-- --:--:-- 19000 404 page not found run task successfully + echo started proxy pid: 9123 started proxy pid: 9123 + echo started proxy at port: 11934 started proxy at port: 11934 + cd /tmp/tidb_cdc_test/http_proxies ++ run_cdc_cli_tso_query 10.233.127.82 2379 + pd_host=10.233.127.82 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://10.233.127.82:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.cli.12603.out cli tso query --pd=http://10.233.127.82:2379 running unsafe_apis status code 200 pass test: show metadata status code 200 status code 200 pass test: delete service_gc_safepoint status code 500 pass test: resolve lock [Mon May 6 13:35:26 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file_large_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file_large_value.1425414256.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x failed proxying s2c rpc error: code = Unavailable desc = client disconnected + tso='449570072419106817 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570072419106817 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449570072419106817 + echo 'query start ts: 449570072419106817' query start ts: 449570072419106817 + run_cdc_server --workdir /tmp/tidb_cdc_test/http_proxies --binary cdc.test [Mon May 6 13:35:27 CST 2024] <<<<<< START cdc server in http_proxies case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.1264512647.out server --log-file /tmp/tidb_cdc_test/http_proxies/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_proxies/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to proxy 127.0.0.1 port 11934 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 11934 (#0) * Server auth using Basic with user 'ticdc' > GET http://127.0.0.1:8300/debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Proxy-Connection: Keep-Alive > < HTTP/1.1 502 Bad Gateway < Date: Mon, 06 May 2024 05:35:27 GMT < Content-Length: 0 < * Connection #0 to host 127.0.0.1 left intact + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled invalid ~~~ running cdc Failed to start cdc, the usage tips should be printed 2nd test case cdc_server_tips success! [Mon May 6 13:35:28 CST 2024] <<<<<< run all test cases cdc_server_tips success! >>>>>> wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc already exists) wait process cdc.test exit for 3-th time... [Pipeline] // cache [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:35:30 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c22ac962-dbad-4aa8-9ff9-f3f4f15292b3 {"id":"c22ac962-dbad-4aa8-9ff9-f3f4f15292b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973727} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c6618c9cc c22ac962-dbad-4aa8-9ff9-f3f4f15292b3 /tidb/cdc/default/default/upstream/7365756012632980229 {"id":7365756012632980229,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c22ac962-dbad-4aa8-9ff9-f3f4f15292b3 {"id":"c22ac962-dbad-4aa8-9ff9-f3f4f15292b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973727} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c6618c9cc c22ac962-dbad-4aa8-9ff9-f3f4f15292b3 /tidb/cdc/default/default/upstream/7365756012632980229 {"id":7365756012632980229,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c22ac962-dbad-4aa8-9ff9-f3f4f15292b3 {"id":"c22ac962-dbad-4aa8-9ff9-f3f4f15292b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973727} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c6618c9cc c22ac962-dbad-4aa8-9ff9-f3f4f15292b3 /tidb/cdc/default/default/upstream/7365756012632980229 {"id":7365756012632980229,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:35:30 CST 2024] <<<<<< run test case http_api_tls_with_user_auth success! >>>>>> [Pipeline] // node [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to proxy 127.0.0.1 port 11934 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 11934 (#0) * Server auth using Basic with user 'ticdc' > GET http://127.0.0.1:8300/debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Proxy-Connection: Keep-Alive > < HTTP/1.1 200 OK < Content-Length: 813 < Content-Type: text/plain; charset=utf-8 < Date: Mon, 06 May 2024 05:35:30 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa3a3caf-b80a-43e9-9f77-97cbe85b1264 {"id":"fa3a3caf-b80a-43e9-9f77-97cbe85b1264","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973727} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2a3d8f4c65d89f26 fa3a3caf-b80a-43e9-9f77-97cbe85b1264 /tidb/cdc/default/default/upstream/7365755934636966444 {"id":7365755934636966444,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.127.82:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa3a3caf-b80a-43e9-9f77-97cbe85b1264 {"id":"fa3a3caf-b80a-43e9-9f77-97cbe85b1264","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973727} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2a3d8f4c65d89f26 fa3a3caf-b80a-43e9-9f77-97cbe85b1264 /tidb/cdc/default/default/upstream/7365755934636966444 {"id":7365755934636966444,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.127.82:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa3a3caf-b80a-43e9-9f77-97cbe85b1264 {"id":"fa3a3caf-b80a-43e9-9f77-97cbe85b1264","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973727} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2a3d8f4c65d89f26 fa3a3caf-b80a-43e9-9f77-97cbe85b1264 /tidb/cdc/default/default/upstream/7365755934636966444 {"id":7365755934636966444,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.127.82:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + echo started cdc server successfully started cdc server successfully + SINK_URI=blackhole:/// + run_cdc_cli changefeed create --start-ts=449570072419106817 --sink-uri=blackhole:/// + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.cli.12708.out cli changefeed create --start-ts=449570072419106817 --sink-uri=blackhole:/// [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ***************** properties ***************** "fieldlengthdistribution"="constant" "fieldlength"="15000" "mysql.db"="consistent_replicate_storage_file_large_value" "dotransactions"="false" "operationcount"="0" "updateproportion"="0" "workload"="core" "readproportion"="0" "recordcount"="100" "fieldcount"="1000" "requestdistribution"="uniform" "readallfields"="true" "insertproportion"="0" "threadcount"="10" "mysql.port"="4000" "mysql.host"="127.0.0.1" "mysql.user"="root" "scanproportion"="0" ********************************************** Create changefeed successfully! ID: b6a7677f-d41a-4f6e-b391-d3d320276570 Info: {"upstream_id":7365755934636966444,"namespace":"default","id":"b6a7677f-d41a-4f6e-b391-d3d320276570","sink_uri":"blackhole:///","create_time":"2024-05-06T13:35:30.972910968+08:00","start_ts":449570072419106817,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570072419106817,"checkpoint_ts":449570072419106817,"checkpoint_time":"2024-05-06 13:35:25.964"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + sleep 5 start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + check + services=($(cat $WORK_DIR/test_proxy.log | xargs -L1 dirname | sort | uniq)) ++ cat /tmp/tidb_cdc_test/http_proxies/test_proxy.log ++ xargs -L1 dirname ++ sort ++ uniq xargs: unmatched double quote; by default quotes are special to xargs unless you use the -0 option + service_type_count=5 + echo 'captured services: ' captured services: + echo . /grpc.health.v1.Health /pdpb.PD /resource_manager.ResourceManager '[2024/05' . /grpc.health.v1.Health /pdpb.PD /resource_manager.ResourceManager [2024/05 + '[' 5 -ge 2 ']' + check_logs /tmp/tidb_cdc_test/http_proxies ++ date + echo '[Mon May 6 13:35:37 CST 2024] <<<<<< run test case http_proxies success! >>>>>>' [Mon May 6 13:35:37 CST 2024] <<<<<< run test case http_proxies success! >>>>>> + stop_tidb_cluster failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Unavailable desc = client disconnected failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Unavailable desc = client disconnected failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/cli_tls_with_auth/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/event_filter/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Run finished, takes 9.398951748s INSERT_ERROR - Takes(s): 8.4, Count: 100, OPS: 11.9, Avg(us): 333964, Min(us): 195715, Max(us): 489067, 95th(us): 428000, 99th(us): 475000 table consistent_replicate_storage_file_large_value.usertable exists table consistent_replicate_storage_file_large_value.check1 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table consistent_replicate_storage_file_large_value.check1 exists table consistent_replicate_storage_file_large_value.t2 not exists for 1-th check, retry later Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319a6c400017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-xwksv-n7b01-mfk0q, pid:24779, start at 2024-05-06 13:35:43.928846801 +0800 CST m=+5.150181305 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:37:43.935 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:35:43.938 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:25:43.938 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319a6c400017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-xwksv-n7b01-mfk0q, pid:24779, start at 2024-05-06 13:35:43.928846801 +0800 CST m=+5.150181305 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:37:43.935 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:35:43.938 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:25:43.938 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319a6e700015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-xwksv-n7b01-mfk0q, pid:24867, start at 2024-05-06 13:35:44.064483919 +0800 CST m=+5.229170058 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:37:44.071 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:35:44.028 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:25:44.028 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/split_region/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table consistent_replicate_storage_file_large_value.t2 not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table consistent_replicate_storage_file_large_value.t2 not exists for 3-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.26119.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449570078048649217 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570078048649217 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file_large_value.t2 not exists for 4-th check, retry later + stop_proxy + kill 9123 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/sequence/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:35:48 CST 2024] <<<<<< run test case sequence success! >>>>>> [Mon May 6 13:35:50 CST 2024] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.2616326165.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file_large_value.t2 not exists for 5-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // withEnv [Pipeline] } table consistent_replicate_storage_file_large_value.t2 not exists for 6-th check, retry later [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:35:53 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7329ac7a-ba78-46fb-91ed-3333ce811bcf {"id":"7329ac7a-ba78-46fb-91ed-3333ce811bcf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973750} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c666f4cfd 7329ac7a-ba78-46fb-91ed-3333ce811bcf /tidb/cdc/default/default/upstream/7365756106918190305 {"id":7365756106918190305,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7329ac7a-ba78-46fb-91ed-3333ce811bcf {"id":"7329ac7a-ba78-46fb-91ed-3333ce811bcf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973750} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c666f4cfd 7329ac7a-ba78-46fb-91ed-3333ce811bcf /tidb/cdc/default/default/upstream/7365756106918190305 {"id":7365756106918190305,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7329ac7a-ba78-46fb-91ed-3333ce811bcf {"id":"7329ac7a-ba78-46fb-91ed-3333ce811bcf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973750} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c666f4cfd 7329ac7a-ba78-46fb-91ed-3333ce811bcf /tidb/cdc/default/default/upstream/7365756106918190305 {"id":7365756106918190305,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.26213.out cli changefeed create --start-ts=449570078048649217 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config /tmp/tidb_cdc_test/split_region/pulsar_test.toml Create changefeed successfully! ID: 6f32cb8a-a64f-481f-a18b-ade06c990f08 Info: {"upstream_id":7365756106918190305,"namespace":"default","id":"6f32cb8a-a64f-481f-a18b-ade06c990f08","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:35:54.005222023+08:00","start_ts":449570078048649217,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570078048649217,"checkpoint_ts":449570078048649217,"checkpoint_time":"2024-05-06 13:35:47.439"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319b09d40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d, pid:39506, start at 2024-05-06 13:35:53.996428107 +0800 CST m=+5.312786209 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:37:54.002 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:35:53.973 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:25:53.973 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319b0b800014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:27333, start at 2024-05-06 13:35:54.1247969 +0800 CST m=+5.207045470 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:37:54.132 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:35:54.130 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:25:54.130 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319b0b800014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:27333, start at 2024-05-06 13:35:54.1247969 +0800 CST m=+5.207045470 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:37:54.132 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:35:54.130 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:25:54.130 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319b0c2c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:27415, start at 2024-05-06 13:35:54.147414374 +0800 CST m=+5.181908282 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:37:54.155 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:35:54.123 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:25:54.123 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/event_filter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/event_filter/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table consistent_replicate_storage_file_large_value.t2 not exists for 7-th check, retry later + set +x table split_region.test1 not exists for 1-th check, retry later [Mon May 6 13:35:57 CST 2024] <<<<<< START cdc server in event_filter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.2866428666.out server --log-file /tmp/tidb_cdc_test/event_filter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/event_filter/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319b09d40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d, pid:39506, start at 2024-05-06 13:35:53.996428107 +0800 CST m=+5.312786209 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:37:54.002 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:35:53.973 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:25:53.973 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319b07e00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d, pid:39594, start at 2024-05-06 13:35:53.850279292 +0800 CST m=+5.087838583 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:37:53.856 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:35:53.848 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:25:53.848 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table consistent_replicate_storage_file_large_value.t2 exists check diff successfully table split_region.test1 exists table split_region.test2 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting TLS PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 13:35:59 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file_large_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file_large_value.1456514567.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table split_region.test2 exists check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:36:00 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5dec28f5-2390-4b67-895d-111da93d7c2c {"id":"5dec28f5-2390-4b67-895d-111da93d7c2c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973757} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c6696eac7 5dec28f5-2390-4b67-895d-111da93d7c2c /tidb/cdc/default/default/upstream/7365756149306166857 {"id":7365756149306166857,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5dec28f5-2390-4b67-895d-111da93d7c2c {"id":"5dec28f5-2390-4b67-895d-111da93d7c2c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973757} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c6696eac7 5dec28f5-2390-4b67-895d-111da93d7c2c /tidb/cdc/default/default/upstream/7365756149306166857 {"id":7365756149306166857,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5dec28f5-2390-4b67-895d-111da93d7c2c {"id":"5dec28f5-2390-4b67-895d-111da93d7c2c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973757} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c6696eac7 5dec28f5-2390-4b67-895d-111da93d7c2c /tidb/cdc/default/default/upstream/7365756149306166857 {"id":7365756149306166857,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.cli.28724.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/event_filter/conf/cf.toml Create changefeed successfully! ID: ba25b412-7567-47f9-a00f-f87da8671ff9 Info: {"upstream_id":7365756149306166857,"namespace":"default","id":"ba25b412-7567-47f9-a00f-f87da8671ff9","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:36:00.735325292+08:00","start_ts":449570081506590724,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["event_filter.*"],"event_filters":[{"matcher":["event_filter.t1"],"ignore_event":["drop table","delete"],"ignore_sql":null,"ignore_insert_value_expr":"id = 2 or city = 'tokyo'","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_truncate"],"ignore_event":["truncate table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_alter"],"ignore_event":["alter table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""}]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570081506590724,"checkpoint_ts":449570081506590724,"checkpoint_time":"2024-05-06 13:36:00.630"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x Starting TLS TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:36:02 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/02498a6b-ab49-42dc-9b52-f6ebcd806680 {UpstreamID:7365756012632980229 Namespace:default ID:02498a6b-ab49-42dc-9b52-f6ebcd806680 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:35:30.22110894 +0800 CST StartTs:449570073501761540 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003425200 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570073528238086} {CheckpointTs:449570076319809560 MinTableBarrierTs:449570081798881283 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570076319809560, checkpointTs: 449570076319809560, state: Preparing span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating span: {table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating span: {table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/84de6518-283b-4b0f-9bbc-916128798037 {"id":"84de6518-283b-4b0f-9bbc-916128798037","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973759} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c6618cb3a 84de6518-283b-4b0f-9bbc-916128798037 /tidb/cdc/default/default/changefeed/info/02498a6b-ab49-42dc-9b52-f6ebcd806680 {"upstream-id":7365756012632980229,"namespace":"default","changefeed-id":"02498a6b-ab49-42dc-9b52-f6ebcd806680","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:35:30.22110894+08:00","start-ts":449570073501761540,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570073528238086} /tidb/cdc/default/default/changefeed/status/02498a6b-ab49-42dc-9b52-f6ebcd806680 {"checkpoint-ts":449570076319809560,"min-table-barrier-ts":449570081798881283,"admin-job-type":0} /tidb/cdc/default/default/task/position/84de6518-283b-4b0f-9bbc-916128798037/02498a6b-ab49-42dc-9b52-f6ebcd806680 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365756012632980229 {"id":7365756012632980229,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/02498a6b-ab49-42dc-9b52-f6ebcd806680 {UpstreamID:7365756012632980229 Namespace:default ID:02498a6b-ab49-42dc-9b52-f6ebcd806680 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:35:30.22110894 +0800 CST StartTs:449570073501761540 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003425200 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570073528238086} {CheckpointTs:449570076319809560 MinTableBarrierTs:449570081798881283 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570076319809560, checkpointTs: 449570076319809560, state: Preparing span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating span: {table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating span: {table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/84de6518-283b-4b0f-9bbc-916128798037 {"id":"84de6518-283b-4b0f-9bbc-916128798037","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973759} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c6618cb3a 84de6518-283b-4b0f-9bbc-916128798037 /tidb/cdc/default/default/changefeed/info/02498a6b-ab49-42dc-9b52-f6ebcd806680 {"upstream-id":7365756012632980229,"namespace":"default","changefeed-id":"02498a6b-ab49-42dc-9b52-f6ebcd806680","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:35:30.22110894+08:00","start-ts":449570073501761540,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570073528238086} /tidb/cdc/default/default/changefeed/status/02498a6b-ab49-42dc-9b52-f6ebcd806680 {"checkpoint-ts":449570076319809560,"min-table-barrier-ts":449570081798881283,"admin-job-type":0} /tidb/cdc/default/default/task/position/84de6518-283b-4b0f-9bbc-916128798037/02498a6b-ab49-42dc-9b52-f6ebcd806680 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365756012632980229 {"id":7365756012632980229,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/02498a6b-ab49-42dc-9b52-f6ebcd806680 {UpstreamID:7365756012632980229 Namespace:default ID:02498a6b-ab49-42dc-9b52-f6ebcd806680 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 13:35:30.22110894 +0800 CST StartTs:449570073501761540 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003425200 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570073528238086} {CheckpointTs:449570076319809560 MinTableBarrierTs:449570081798881283 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449570076319809560, checkpointTs: 449570076319809560, state: Preparing span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating span: {table_id:115,start_key:7480000000000000ff735f720000000000fa,end_key:7480000000000000ff735f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating span: {table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}, resolvedTs: 449570081798881283, checkpointTs: 449570076319809560, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/84de6518-283b-4b0f-9bbc-916128798037 {"id":"84de6518-283b-4b0f-9bbc-916128798037","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973759} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c6618cb3a 84de6518-283b-4b0f-9bbc-916128798037 /tidb/cdc/default/default/changefeed/info/02498a6b-ab49-42dc-9b52-f6ebcd806680 {"upstream-id":7365756012632980229,"namespace":"default","changefeed-id":"02498a6b-ab49-42dc-9b52-f6ebcd806680","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T13:35:30.22110894+08:00","start-ts":449570073501761540,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570073528238086} /tidb/cdc/default/default/changefeed/status/02498a6b-ab49-42dc-9b52-f6ebcd806680 {"checkpoint-ts":449570076319809560,"min-table-barrier-ts":449570081798881283,"admin-job-type":0} /tidb/cdc/default/default/task/position/84de6518-283b-4b0f-9bbc-916128798037/02498a6b-ab49-42dc-9b52-f6ebcd806680 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365756012632980229 {"id":7365756012632980229,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) count(*) 0 Starting TLS TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table event_filter.t1 does not exists table event_filter.t1 exists table event_filter.t_normal not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table event_filter.t_normal exists table event_filter.t_truncate not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table event_filter.t_truncate exists table event_filter.t_alter not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319bfb200015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-mlls2-941gl-6sq5d, pid:41184, start at 2024-05-06 13:36:09.454885311 +0800 CST m=+5.036444094 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:38:09.461 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:36:09.467 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:26:09.467 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + pd_host=127.0.0.1 + pd_port=2579 + is_tls=true + '[' true == true ']' ++ run_cdc_cli tso query --pd=https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.41244.out cli tso query --pd=https://127.0.0.1:2579 table event_filter.t_alter exists check diff failed 1-th time, retry later + set +x + tso='449570084203266049 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570084203266049 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 13:36:12 CST 2024] <<<<<< START cdc server in cli_tls_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.4129641298.out server --log-file /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_cli_tls_with_auth_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_data_cli_tls_with_auth_tls1 --cluster-id default --config /tmp/tidb_cdc_test/cli_tls_with_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2581 0 --:--:-- --:--:-- --:--:-- 2600 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 13:35:59.928","puller_resolved_ts":"2024-05-06 13:35:52.928","last_synced_ts":"2024-05-06 13:33:43.678","now_ts":"2024-05-06 13:36:01.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '13:35:59.928","puller_resolved_ts":"2024-05-06' '13:35:52.928","last_synced_ts":"2024-05-06' '13:33:43.678","now_ts":"2024-05-06' '13:36:01.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 16105 7.1 0.0 13464152 143412 ? Sl 13:33 0:11 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 16173 4.8 0.0 13717716 138756 ? Sl 13:33 0:07 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1714973766.2959895,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003722700/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714973766.296046,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714973766.727905,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010b56c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714973766.7279482,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714973767.1431017,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0021e3880/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714973767.14315,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T13:36:10.982209+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f44fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:36:10.98835+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001070000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:36:11.06882+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e30e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table event_filter.finish_mark exists check diff failed 1-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:36:15 CST 2024] <<<<<< run test case split_region success! >>>>>> [Pipeline] // dir Post stage [Pipeline] sh check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:36:15 GMT < Content-Length: 1209 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e5adfbb6-a7fe-484b-9227-8d9beb585519 {"id":"e5adfbb6-a7fe-484b-9227-8d9beb585519","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973773} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c66d67f84 e5adfbb6-a7fe-484b-9227-8d9beb585519 /tidb/cdc/default/default/upstream/7365756219986826468 {"id":7365756219986826468,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e5adfbb6-a7fe-484b-9227-8d9beb585519 {"id":"e5adfbb6-a7fe-484b-9227-8d9beb585519","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973773} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c66d67f84 e5adfbb6-a7fe-484b-9227-8d9beb585519 /tidb/cdc/default/default/upstream/7365756219986826468 {"id":7365756219986826468,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e5adfbb6-a7fe-484b-9227-8d9beb585519 {"id":"e5adfbb6-a7fe-484b-9227-8d9beb585519","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973773} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4c66d67f84 e5adfbb6-a7fe-484b-9227-8d9beb585519 /tidb/cdc/default/default/upstream/7365756219986826468 {"id":7365756219986826468,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x + ls /tmp/tidb_cdc_test/ consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value cov.consistent_replicate_ddl.26322634.out cov.consistent_replicate_ddl.28752877.out cov.consistent_replicate_gbk.56755677.out cov.consistent_replicate_gbk.58825884.out cov.consistent_replicate_nfs.84648466.out cov.consistent_replicate_storage_file.1114611148.out cov.consistent_replicate_storage_file.1151811520.out cov.consistent_replicate_storage_file_large_value.1425414256.out sql_res.consistent_replicate_ddl.txt sql_res.consistent_replicate_gbk.txt sql_res.consistent_replicate_nfs.txt sql_res.consistent_replicate_storage_file.txt sql_res.consistent_replicate_storage_file_large_value.txt ++ find /tmp/tidb_cdc_test/ -type f -name '*.log' + tar -cvzf log-G02.tar.gz /tmp/tidb_cdc_test/consistent_replicate_ddl/tidb_down.log /tmp/tidb_cdc_test/consistent_replicate_ddl/down_pd.log /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server2.log /tmp/tidb_cdc_test/consistent_replicate_ddl/tikv3.log /tmp/tidb_cdc_test/consistent_replicate_ddl/tikv1.log /tmp/tidb_cdc_test/consistent_replicate_ddl/tikv_down.log /tmp/tidb_cdc_test/consistent_replicate_ddl/tidb_other.log /tmp/tidb_cdc_test/consistent_replicate_ddl/stdoutconsistent_replicate_ddl.server2.log /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server1.log /tmp/tidb_cdc_test/consistent_replicate_ddl/stdoutconsistent_replicate_ddl.server1.log /tmp/tidb_cdc_test/consistent_replicate_ddl/tidb.log /tmp/tidb_cdc_test/consistent_replicate_ddl/pd1.log /tmp/tidb_cdc_test/consistent_replicate_ddl/sync_diff_inspector.log /tmp/tidb_cdc_test/consistent_replicate_ddl/tikv2.log /tmp/tidb_cdc_test/consistent_replicate_ddl/tidb-slow.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tidb_down.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/pd1/region-meta/000001.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/pd1/hot-region/000001.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/down_pd.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/sync_diff/output/sync_diff.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tikv2/db/000005.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tikv3.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/stdout.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tikv1.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/proxy.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/server.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/error.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tikv_down.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tidb_other.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tikv_down/db/000005.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/down_pd/region-meta/000001.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/down_pd/hot-region/000001.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tikv1/db/000005.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tidb.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/pd1.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/sync_diff_inspector.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tikv2.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tidb-slow.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tikv3/db/000005.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo/127.0.0.1:8300_02498a6b-ab49-42dc-9b52-f6ebcd806680_ddl_449570076319809560_e2f506cc-ec4c-415b-921d-eecf99741903.log /tmp/tidb_cdc_test/consistent_replicate_nfs/tidb_down.log /tmp/tidb_cdc_test/consistent_replicate_nfs/down_pd.log /tmp/tidb_cdc_test/consistent_replicate_nfs/tikv3.log /tmp/tidb_cdc_test/consistent_replicate_nfs/stdout.log /tmp/tidb_cdc_test/consistent_replicate_nfs/tikv1.log /tmp/tidb_cdc_test/consistent_replicate_nfs/tikv_down.log /tmp/tidb_cdc_test/consistent_replicate_nfs/tidb_other.log /tmp/tidb_cdc_test/consistent_replicate_nfs/tidb.log /tmp/tidb_cdc_test/consistent_replicate_nfs/pd1.log /tmp/tidb_cdc_test/consistent_replicate_nfs/sync_diff_inspector.log /tmp/tidb_cdc_test/consistent_replicate_nfs/tikv2.log /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc.log /tmp/tidb_cdc_test/consistent_replicate_nfs/tidb-slow.log /tmp/tidb_cdc_test/consistent_replicate_gbk/tidb_down.log /tmp/tidb_cdc_test/consistent_replicate_gbk/down_pd.log /tmp/tidb_cdc_test/consistent_replicate_gbk/tikv3.log /tmp/tidb_cdc_test/consistent_replicate_gbk/stdout.log /tmp/tidb_cdc_test/consistent_replicate_gbk/tikv1.log /tmp/tidb_cdc_test/consistent_replicate_gbk/tikv_down.log /tmp/tidb_cdc_test/consistent_replicate_gbk/tidb_other.log /tmp/tidb_cdc_test/consistent_replicate_gbk/tidb.log /tmp/tidb_cdc_test/consistent_replicate_gbk/pd1.log /tmp/tidb_cdc_test/consistent_replicate_gbk/sync_diff_inspector.log /tmp/tidb_cdc_test/consistent_replicate_gbk/tikv2.log /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log /tmp/tidb_cdc_test/consistent_replicate_gbk/tidb-slow.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/tidb_down.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/down_pd.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/tikv3.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/stdout.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/tikv1.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/tikv_down.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/tidb_other.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/tidb.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/pd1.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/sync_diff_inspector.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/tikv2.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/tidb-slow.log tar: Removing leading `/' from member names /tmp/tidb_cdc_test/consistent_replicate_ddl/tidb_down.log /tmp/tidb_cdc_test/consistent_replicate_ddl/down_pd.log /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server2.log /tmp/tidb_cdc_test/consistent_replicate_ddl/tikv3.log /tmp/tidb_cdc_test/consistent_replicate_ddl/tikv1.log /tmp/tidb_cdc_test/consistent_replicate_ddl/tikv_down.log /tmp/tidb_cdc_test/consistent_replicate_ddl/tidb_other.log /tmp/tidb_cdc_test/consistent_replicate_ddl/stdoutconsistent_replicate_ddl.server2.log /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server1.log /tmp/tidb_cdc_test/consistent_replicate_ddl/stdoutconsistent_replicate_ddl.server1.log /tmp/tidb_cdc_test/consistent_replicate_ddl/tidb.log /tmp/tidb_cdc_test/consistent_replicate_ddl/pd1.log /tmp/tidb_cdc_test/consistent_replicate_ddl/sync_diff_inspector.log /tmp/tidb_cdc_test/consistent_replicate_ddl/tikv2.log {"level":"warn","ts":"2024-05-06T13:36:16.983307+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f44fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:36:16.989076+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001070000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} /tmp/tidb_cdc_test/consistent_replicate_ddl/tidb-slow.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tidb_down.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/pd1/region-meta/000001.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/pd1/hot-region/000001.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/down_pd.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/sync_diff/output/sync_diff.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tikv2/db/000005.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tikv3.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/stdout.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tikv1.log {"level":"warn","ts":"2024-05-06T13:36:17.070139+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e30e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/proxy.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/server.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/error.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tikv_down.log check diff successfully /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tidb_other.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tikv_down/db/000005.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/down_pd/region-meta/000001.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/down_pd/hot-region/000001.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tikv1/db/000005.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tidb.log wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.41364.out cli changefeed create --start-ts=449570084203266049 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365756219986826468,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:36:18.433947545+08:00","start_ts":449570084203266049,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570084203266049,"checkpoint_ts":449570084203266049,"checkpoint_time":"2024-05-06 13:36:10.917"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:36:18 CST 2024] <<<<<< run test case event_filter success! >>>>>> + set +x table test.simple not exists for 1-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed table test.simple exists table test.`simple-dash` not exists for 1-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0{"level":"warn","ts":"2024-05-06T13:36:22.984647+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f44fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:36:22.99104+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001070000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:36:23.071565+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e30e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table test.`simple-dash` exists + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365756219986826468, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449570084622434318, "checkpoint_time": "2024-05-06 13:36:12.516", "error": null }' + echo '{ "upstream_id": 7365756219986826468, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449570084622434318, "checkpoint_time": "2024-05-06 13:36:12.516", "error": null }' { "upstream_id": 7365756219986826468, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449570084622434318, "checkpoint_time": "2024-05-06 13:36:12.516", "error": null } ++ echo '{' '"upstream_id":' 7365756219986826468, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449570084622434318, '"checkpoint_time":' '"2024-05-06' '13:36:12.516",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365756219986826468, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449570084622434318, '"checkpoint_time":' '"2024-05-06' '13:36:12.516",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: https://127.0.0.1:2579 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1779/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir Post stage Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.41604.out cli changefeed --changefeed-id custom-changefeed-name pause [Pipeline] sh PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + ls /tmp/tidb_cdc_test/ cdc cov.cdc.55095511.out cov.cdc.cli.5560.out cov.kv_client_stream_reconnect.26332635.out cov.split_region.2616326165.out cov.split_region.cli.26119.out cov.split_region.cli.26213.out kv_client_stream_reconnect split_region sql_res.kv_client_stream_reconnect.txt sql_res.split_region.txt ++ find /tmp/tidb_cdc_test/ -type f -name '*.log' + tar -cvzf log-G07.tar.gz /tmp/tidb_cdc_test/split_region/tikv3/db/000005.log /tmp/tidb_cdc_test/split_region/down_pd.log /tmp/tidb_cdc_test/split_region/tikv3.log /tmp/tidb_cdc_test/split_region/tikv2.log /tmp/tidb_cdc_test/split_region/tikv_down.log /tmp/tidb_cdc_test/split_region/tikv_down/db/000005.log /tmp/tidb_cdc_test/split_region/tikv1/db/000005.log /tmp/tidb_cdc_test/split_region/cdc.log /tmp/tidb_cdc_test/split_region/stdout.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/split_region/tikv2/db/000005.log /tmp/tidb_cdc_test/split_region/tikv1.log /tmp/tidb_cdc_test/split_region/tidb.log /tmp/tidb_cdc_test/split_region/pd1.log /tmp/tidb_cdc_test/split_region/tidb_other.log /tmp/tidb_cdc_test/split_region/sync_diff/output/sync_diff.log /tmp/tidb_cdc_test/split_region/tidb-slow.log /tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log /tmp/tidb_cdc_test/split_region/tiflash/log/error.log /tmp/tidb_cdc_test/split_region/tiflash/log/server.log /tmp/tidb_cdc_test/split_region/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/split_region/sync_diff_inspector.log /tmp/tidb_cdc_test/split_region/pd1/region-meta/000001.log /tmp/tidb_cdc_test/split_region/pd1/hot-region/000001.log /tmp/tidb_cdc_test/split_region/down_pd/region-meta/000001.log /tmp/tidb_cdc_test/split_region/down_pd/hot-region/000001.log /tmp/tidb_cdc_test/split_region/tidb_down.log /tmp/tidb_cdc_test/cdc/tester.log /tmp/tidb_cdc_test/cdc/down_pd.log /tmp/tidb_cdc_test/cdc/tikv3.log /tmp/tidb_cdc_test/cdc/tikv2.log /tmp/tidb_cdc_test/cdc/tikv_down.log /tmp/tidb_cdc_test/cdc/cdc.log /tmp/tidb_cdc_test/cdc/stdout.log /tmp/tidb_cdc_test/cdc/tikv1.log /tmp/tidb_cdc_test/cdc/tidb.log /tmp/tidb_cdc_test/cdc/pd1.log /tmp/tidb_cdc_test/cdc/tidb_other.log /tmp/tidb_cdc_test/cdc/tidb-slow.log /tmp/tidb_cdc_test/cdc/tidb_down.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/down_pd.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tikv3.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tikv2.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tikv_down.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/stdout.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tikv1.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tidb.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/pd1.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tidb_other.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tidb-slow.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/sync_diff_inspector.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tidb_down.log tar: Removing leading `/' from member names /tmp/tidb_cdc_test/split_region/tikv3/db/000005.log /tmp/tidb_cdc_test/split_region/down_pd.log /tmp/tidb_cdc_test/split_region/tikv3.log /tmp/tidb_cdc_test/split_region/tikv2.log /tmp/tidb_cdc_test/split_region/tikv_down.log /tmp/tidb_cdc_test/split_region/tikv_down/db/000005.log /tmp/tidb_cdc_test/split_region/tikv1/db/000005.log /tmp/tidb_cdc_test/split_region/cdc.log /tmp/tidb_cdc_test/split_region/stdout.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/split_region/tikv2/db/000005.log /tmp/tidb_cdc_test/split_region/tikv1.log /tmp/tidb_cdc_test/split_region/tidb.log /tmp/tidb_cdc_test/split_region/pd1.log /tmp/tidb_cdc_test/split_region/tidb_other.log /tmp/tidb_cdc_test/split_region/sync_diff/output/sync_diff.log /tmp/tidb_cdc_test/split_region/tidb-slow.log /tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log /tmp/tidb_cdc_test/split_region/tiflash/log/error.log /tmp/tidb_cdc_test/split_region/tiflash/log/server.log /tmp/tidb_cdc_test/split_region/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/split_region/sync_diff_inspector.log /tmp/tidb_cdc_test/split_region/pd1/region-meta/000001.log /tmp/tidb_cdc_test/split_region/pd1/hot-region/000001.log /tmp/tidb_cdc_test/split_region/down_pd/region-meta/000001.log /tmp/tidb_cdc_test/split_region/down_pd/hot-region/000001.log /tmp/tidb_cdc_test/split_region/tidb_down.log /tmp/tidb_cdc_test/cdc/tester.log /tmp/tidb_cdc_test/cdc/down_pd.log /tmp/tidb_cdc_test/cdc/tikv3.log /tmp/tidb_cdc_test/cdc/tikv2.log /tmp/tidb_cdc_test/cdc/tikv_down.log + set +x /tmp/tidb_cdc_test/cdc/cdc.log =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/sql_mode/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/sql_mode/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/sql_mode ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/sql_mode + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/sql_mode/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/sql_mode + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + rm -rf /tmp/tidb_cdc_test/sql_mode + mkdir -p /tmp/tidb_cdc_test/sql_mode + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... /tmp/tidb_cdc_test/cdc/stdout.log /tmp/tidb_cdc_test/cdc/tikv1.log 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0{"level":"warn","ts":"2024-05-06T13:36:28.985455+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f44fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:36:28.992693+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001070000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:36:29.072832+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e30e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} /tmp/tidb_cdc_test/cdc/tidb.log /tmp/tidb_cdc_test/cdc/pd1.log /tmp/tidb_cdc_test/cdc/tidb_other.log /tmp/tidb_cdc_test/cdc/tidb-slow.log /tmp/tidb_cdc_test/cdc/tidb_down.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/down_pd.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tikv3.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tikv2.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tikv_down.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/pd1.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/sync_diff_inspector.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tikv2.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tidb-slow.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tikv3/db/000005.log /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo/127.0.0.1:8300_02498a6b-ab49-42dc-9b52-f6ebcd806680_ddl_449570076319809560_e2f506cc-ec4c-415b-921d-eecf99741903.log /tmp/tidb_cdc_test/consistent_replicate_nfs/tidb_down.log /tmp/tidb_cdc_test/consistent_replicate_nfs/down_pd.log /tmp/tidb_cdc_test/consistent_replicate_nfs/tikv3.log /tmp/tidb_cdc_test/consistent_replicate_nfs/stdout.log /tmp/tidb_cdc_test/consistent_replicate_nfs/tikv1.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/stdout.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tikv1.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tidb.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/pd1.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tidb_other.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tidb-slow.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/sync_diff_inspector.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tidb_down.log + ls -alh log-G07.tar.gz -rw-r--r--. 1 jenkins jenkins 19M May 6 13:36 log-G07.tar.gz [Pipeline] archiveArtifacts Archiving artifacts + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365756219986826468, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449570088017199113, "checkpoint_time": "2024-05-06 13:36:25.466", "error": null }' + echo '{ "upstream_id": 7365756219986826468, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449570088017199113, "checkpoint_time": "2024-05-06 13:36:25.466", "error": null }' { "upstream_id": 7365756219986826468, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449570088017199113, "checkpoint_time": "2024-05-06 13:36:25.466", "error": null } ++ echo '{' '"upstream_id":' 7365756219986826468, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449570088017199113, '"checkpoint_time":' '"2024-05-06' '13:36:25.466",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365756219986826468, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449570088017199113, '"checkpoint_time":' '"2024-05-06' '13:36:25.466",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.41705.out cli changefeed update --pd=https://127.0.0.1:2579 --config=/tmp/tidb_cdc_test/cli_tls_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0{"level":"warn","ts":"2024-05-06T13:36:30.973623+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f44fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-06T13:36:30.973665+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T13:36:30.981235+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001070000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-06T13:36:30.981271+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T13:36:31.06084+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e30e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-06T13:36:31.060906+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7365756219986826468,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:36:18.433947545+08:00","start_ts":449570084203266049,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":0,"checkpoint_ts":449570088017199113,"checkpoint_time":"2024-05-06 13:36:25.466"} PASS coverage: 2.8% of statements in github.com/pingcap/tiflow/... /tmp/tidb_cdc_test/consistent_replicate_nfs/tikv_down.log Recording fingerprints [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.41740.out cli changefeed --changefeed-id custom-changefeed-name resume Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0{"level":"warn","ts":"2024-05-06T13:36:34.987228+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f44fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:36:34.994273+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001070000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:36:35.074226+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e30e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} + set +x Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... /tmp/tidb_cdc_test/consistent_replicate_nfs/tidb_other.log /tmp/tidb_cdc_test/consistent_replicate_nfs/tidb.log /tmp/tidb_cdc_test/consistent_replicate_nfs/pd1.log /tmp/tidb_cdc_test/consistent_replicate_nfs/sync_diff_inspector.log /tmp/tidb_cdc_test/consistent_replicate_nfs/tikv2.log /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc.log Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /tmp/tidb_cdc_test/consistent_replicate_nfs/tidb-slow.log /tmp/tidb_cdc_test/consistent_replicate_gbk/tidb_down.log /tmp/tidb_cdc_test/consistent_replicate_gbk/down_pd.log /tmp/tidb_cdc_test/consistent_replicate_gbk/tikv3.log /tmp/tidb_cdc_test/consistent_replicate_gbk/stdout.log /tmp/tidb_cdc_test/consistent_replicate_gbk/tikv1.log /tmp/tidb_cdc_test/consistent_replicate_gbk/tikv_down.log /tmp/tidb_cdc_test/consistent_replicate_gbk/tidb_other.log /tmp/tidb_cdc_test/consistent_replicate_gbk/tidb.log /tmp/tidb_cdc_test/consistent_replicate_gbk/pd1.log /tmp/tidb_cdc_test/consistent_replicate_gbk/sync_diff_inspector.log /tmp/tidb_cdc_test/consistent_replicate_gbk/tikv2.log /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log /tmp/tidb_cdc_test/consistent_replicate_gbk/tidb-slow.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/tidb_down.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/down_pd.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/tikv3.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365756219986826468, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449570091176296450, "checkpoint_time": "2024-05-06 13:36:37.517", "error": null }' + echo '{ "upstream_id": 7365756219986826468, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449570091176296450, "checkpoint_time": "2024-05-06 13:36:37.517", "error": null }' { "upstream_id": 7365756219986826468, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449570091176296450, "checkpoint_time": "2024-05-06 13:36:37.517", "error": null } ++ echo '{' '"upstream_id":' 7365756219986826468, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449570091176296450, '"checkpoint_time":' '"2024-05-06' '13:36:37.517",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365756219986826468, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449570091176296450, '"checkpoint_time":' '"2024-05-06' '13:36:37.517",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.41836.out cli changefeed --changefeed-id custom-changefeed-name remove Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 449570091438440450 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/ PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0{"level":"warn","ts":"2024-05-06T13:36:40.989124+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f44fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:36:40.994961+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001070000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:36:41.075698+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e30e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0{"level":"warn","ts":1714973801.2972949,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003722700/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714973801.2973356,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714973801.728907,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010b56c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714973801.7289436,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0{"level":"warn","ts":1714973802.1443386,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0021e3880/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714973802.144375,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319dea980003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:29939, start at 2024-05-06 13:36:41.126783373 +0800 CST m=+5.160671839 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:38:41.133 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:36:41.126 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:26:41.126 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319dea980003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:29939, start at 2024-05-06 13:36:41.126783373 +0800 CST m=+5.160671839 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:38:41.133 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:36:41.126 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:26:41.126 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319deb3c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:30030, start at 2024-05-06 13:36:41.170789089 +0800 CST m=+5.138271770 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:38:41.177 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:36:41.167 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:26:41.167 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sql_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sql_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + trap stop_tidb_cluster EXIT + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 3306 + cd /tmp/tidb_cdc_test/sql_mode ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.31348.out cli tso query --pd=http://127.0.0.1:2379 changefeed count 0 check pass, pd_addr: https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.41921.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365756219986826468,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T13:36:44.680716078+08:00","start_ts":449570092998197255,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570092998197255,"checkpoint_ts":449570092998197255,"checkpoint_time":"2024-05-06 13:36:44.467"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + set +x + tso='449570093039616001 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570093039616001 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449570093039616001 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Mon May 6 13:36:46 CST 2024] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.3139031392.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 9830 0 --:--:-- --:--:-- --:--:-- 9904 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 13:34:24.684","puller_resolved_ts":"2024-05-06 13:34:24.684","last_synced_ts":"2024-05-06 13:34:17.884","now_ts":"2024-05-06 13:36:35.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '13:34:24.684","puller_resolved_ts":"2024-05-06' '13:34:24.684","last_synced_ts":"2024-05-06' '13:34:17.884","now_ts":"2024-05-06' '13:36:35.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '13:34:24.684","puller_resolved_ts":"2024-05-06' '13:34:24.684","last_synced_ts":"2024-05-06' '13:34:17.884","now_ts":"2024-05-06' '13:36:35.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0{"level":"warn","ts":"2024-05-06T13:36:46.990642+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f44fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:36:46.995994+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001070000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T13:36:47.076482+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e30e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365756219986826468, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449570094059618307, "checkpoint_time": "2024-05-06 13:36:48.516", "error": null }' + echo '{ "upstream_id": 7365756219986826468, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449570094059618307, "checkpoint_time": "2024-05-06 13:36:48.516", "error": null }' { "upstream_id": 7365756219986826468, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449570094059618307, "checkpoint_time": "2024-05-06 13:36:48.516", "error": null } ++ echo '{' '"upstream_id":' 7365756219986826468, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449570094059618307, '"checkpoint_time":' '"2024-05-06' '13:36:48.516",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365756219986826468, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449570094059618307, '"checkpoint_time":' '"2024-05-06' '13:36:48.516",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:36:49 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d1c0ca7d-a018-45a3-8fc2-1afe545bec2d {"id":"d1c0ca7d-a018-45a3-8fc2-1afe545bec2d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973806} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c67548ad1 d1c0ca7d-a018-45a3-8fc2-1afe545bec2d /tidb/cdc/default/default/upstream/7365756350300007855 {"id":7365756350300007855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d1c0ca7d-a018-45a3-8fc2-1afe545bec2d {"id":"d1c0ca7d-a018-45a3-8fc2-1afe545bec2d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973806} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c67548ad1 d1c0ca7d-a018-45a3-8fc2-1afe545bec2d /tidb/cdc/default/default/upstream/7365756350300007855 {"id":7365756350300007855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d1c0ca7d-a018-45a3-8fc2-1afe545bec2d {"id":"d1c0ca7d-a018-45a3-8fc2-1afe545bec2d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973806} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c67548ad1 d1c0ca7d-a018-45a3-8fc2-1afe545bec2d /tidb/cdc/default/default/upstream/7365756350300007855 {"id":7365756350300007855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449570093039616001 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.31450.out cli changefeed create --start-ts=449570093039616001 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365756350300007855,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T13:36:49.638926886+08:00","start_ts":449570093039616001,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570093039616001,"checkpoint_ts":449570093039616001,"checkpoint_time":"2024-05-06 13:36:44.625"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' ++ jq -r .error_code + set +x + run_sql 'use test; create table t1(id bigint primary key, a text, b text as ((regexp_replace(a, '\''^[1-9]\d{9,29}$'\'', '\''aaaaa'\''))), c text); insert into t1 (id, a, c) values(1,123456, '\''ab\\\\c'\''); insert into t1 (id, a, c) values(2,1234567890123, '\''ab\\c'\'');' 127.0.0.1 4000 + '[' mysql == mysql ']' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.42054.out cli unsafe delete-service-gc-safepoint Confirm that you know what this command will do and use it at your own risk [Y/N] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.42088.out cli unsafe reset --no-confirm --pd=https://127.0.0.1:2579 table test.t1 exists + sleep 10 reset and all metadata truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.42217.out cli unsafe resolve-lock --region=10 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319f01200012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57, pid:15167, start at 2024-05-06 13:36:58.982444745 +0800 CST m=+5.195095883 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:38:58.991 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:36:58.952 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:26:58.952 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319f01200012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57, pid:15167, start at 2024-05-06 13:36:58.982444745 +0800 CST m=+5.195095883 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:38:58.991 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:36:58.952 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:26:58.952 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319f01d40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-rqdbn-zml25-g1z57, pid:15249, start at 2024-05-06 13:36:59.016071166 +0800 CST m=+5.183514552 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:38:59.022 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:36:58.997 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:26:58.997 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.42248.out cli unsafe resolve-lock --region=10 --ts=449570095815983108 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + run_case_with_unavailable_tikv conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 1670 47 --:--:-- --:--:-- --:--:-- 1682 { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.16543.out cli tso query --pd=http://127.0.0.1:2379 + run_sql 'SELECT * from test.t1' 127.0.0.1 3306 + check_contains 'b: 123456' + check_contains 'b: aaaaa' + check_contains 'c: ab\\\\c' + check_contains 'c: ab\\c' + stop_tidb_cluster + set +x + tso='449570097699487745 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570097699487745 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449570097699487745 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Mon May 6 13:37:03 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1658516587.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 1384 0 --:--:-- --:--:-- --:--:-- 1389 { "version": "v8.2.0-alpha-22-gd72d260c0", "git_hash": "d72d260c017170db94deda602246fcddfea2a4a1", "id": "eb50e88e-b87c-4c77-9666-7c0637eae38f", "pid": 41301, "is_owner": true }wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 13:37:06 CST 2024] <<<<<< run test case cli_tls_with_auth success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:37:06 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef4569d5-6053-450b-8000-b2a023e39cb3 {"id":"ef4569d5-6053-450b-8000-b2a023e39cb3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973824} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c679644cd ef4569d5-6053-450b-8000-b2a023e39cb3 /tidb/cdc/default/default/upstream/7365756429835161963 {"id":7365756429835161963,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef4569d5-6053-450b-8000-b2a023e39cb3 {"id":"ef4569d5-6053-450b-8000-b2a023e39cb3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973824} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c679644cd ef4569d5-6053-450b-8000-b2a023e39cb3 /tidb/cdc/default/default/upstream/7365756429835161963 {"id":7365756429835161963,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef4569d5-6053-450b-8000-b2a023e39cb3 {"id":"ef4569d5-6053-450b-8000-b2a023e39cb3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973824} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c679644cd ef4569d5-6053-450b-8000-b2a023e39cb3 /tidb/cdc/default/default/upstream/7365756429835161963 {"id":7365756429835161963,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449570097699487745 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.16639.out cli changefeed create --start-ts=449570097699487745 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Verifying downstream PD is started... Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365756429835161963,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T13:37:07.398031187+08:00","start_ts":449570097699487745,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570097699487745,"checkpoint_ts":449570097699487745,"checkpoint_time":"2024-05-06 13:37:02.401"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later /tmp/tidb_cdc_test/consistent_replicate_storage_file/stdout.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/tikv1.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/tikv_down.log Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t1 exists + sleep 5 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319ff5dc001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd, pid:19480, start at 2024-05-06 13:37:14.66323162 +0800 CST m=+5.023088227 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:39:14.670 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:37:14.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:27:14.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319ff5dc001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd, pid:19480, start at 2024-05-06 13:37:14.66323162 +0800 CST m=+5.023088227 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:39:14.670 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:37:14.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:27:14.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319ff9a8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-kkw9l-ghxcj-b5szd, pid:19554, start at 2024-05-06 13:37:14.869874417 +0800 CST m=+5.145684283 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:39:14.878 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:37:14.858 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:27:14.858 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_simple_basic/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:37:16 CST 2024] <<<<<< run test case kafka_simple_basic success! >>>>>> + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 15163 3.8 0.0 2530696 192796 ? Sl 13:36 0:00 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1714973813779029936.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb.log jenkins 15167 12.4 0.0 2738896 252484 ? Sl 13:36 0:02 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1714973813781918103.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb_other.log jenkins 15249 12.3 0.0 2640320 259756 ? Sl 13:36 0:02 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1714973813826609030.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 3135 0 --:--:-- --:--:-- --:--:-- 3155 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 13:37:14.851","puller_resolved_ts":"2024-05-06 13:37:08.901","last_synced_ts":"2024-05-06 13:37:09.401","now_ts":"2024-05-06 13:37:16.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '13:37:14.851","puller_resolved_ts":"2024-05-06' '13:37:08.901","last_synced_ts":"2024-05-06' '13:37:09.401","now_ts":"2024-05-06' '13:37:16.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '13:37:14.851","puller_resolved_ts":"2024-05-06' '13:37:08.901","last_synced_ts":"2024-05-06' '13:37:09.401","now_ts":"2024-05-06' '13:37:16.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.20873.out cli tso query --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:37:19 CST 2024] <<<<<< run test case kafka_simple_basic_avro success! >>>>>> + set +x + tso='449570101858140162 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570101858140162 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449570101858140162 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Mon May 6 13:37:19 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2091320915.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:37:22 CST 2024] <<<<<< run test case kafka_simple_handle_key_only success! >>>>>> Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:37:22 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cd515e4e-0704-42f4-8c1c-02d84ca54bda {"id":"cd515e4e-0704-42f4-8c1c-02d84ca54bda","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973840} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c67d215ce cd515e4e-0704-42f4-8c1c-02d84ca54bda /tidb/cdc/default/default/upstream/7365756492353772084 {"id":7365756492353772084,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cd515e4e-0704-42f4-8c1c-02d84ca54bda {"id":"cd515e4e-0704-42f4-8c1c-02d84ca54bda","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973840} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c67d215ce cd515e4e-0704-42f4-8c1c-02d84ca54bda /tidb/cdc/default/default/upstream/7365756492353772084 {"id":7365756492353772084,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cd515e4e-0704-42f4-8c1c-02d84ca54bda {"id":"cd515e4e-0704-42f4-8c1c-02d84ca54bda","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973840} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c67d215ce cd515e4e-0704-42f4-8c1c-02d84ca54bda /tidb/cdc/default/default/upstream/7365756492353772084 {"id":7365756492353772084,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449570101858140162 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.20974.out cli changefeed create --start-ts=449570101858140162 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365756492353772084,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T13:37:23.265909449+08:00","start_ts":449570101858140162,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570101858140162,"checkpoint_ts":449570101858140162,"checkpoint_time":"2024-05-06 13:37:18.265"} PASS Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:37:25 CST 2024] <<<<<< run test case kafka_simple_handle_key_only_avro success! >>>>>> + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t1 exists + sleep 5 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319dea980003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:29939, start at 2024-05-06 13:36:41.126783373 +0800 CST m=+5.160671839 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:38:41.133 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:36:41.126 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:26:41.126 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319dea980003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:29939, start at 2024-05-06 13:36:41.126783373 +0800 CST m=+5.160671839 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:38:41.133 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:36:41.126 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:26:41.126 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319deb3c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:30030, start at 2024-05-06 13:36:41.170789089 +0800 CST m=+5.138271770 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:38:41.177 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:36:41.167 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:26:41.167 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 2 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_simple_claim_check/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:37:28 CST 2024] <<<<<< run test case kafka_simple_claim_check success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:37:31 CST 2024] <<<<<< run test case kafka_simple_claim_check_avro success! >>>>>> + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 18901 23.3 0.5 4422248 2195456 ? Sl 13:37 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 18902 30.6 0.5 4456040 2261668 ? Sl 13:37 0:07 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 18903 23.6 0.5 4430952 2215416 ? Sl 13:37 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 18905 29.3 0.5 4458600 2253424 ? Sl 13:37 0:07 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2460 0 --:--:-- --:--:-- --:--:-- 2479 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 13:37:31.715","puller_resolved_ts":"2024-05-06 13:37:24.715","last_synced_ts":"2024-05-06 13:37:25.215","now_ts":"2024-05-06 13:37:32.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '13:37:31.715","puller_resolved_ts":"2024-05-06' '13:37:24.715","last_synced_ts":"2024-05-06' '13:37:25.215","now_ts":"2024-05-06' '13:37:32.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '13:37:31.715","puller_resolved_ts":"2024-05-06' '13:37:24.715","last_synced_ts":"2024-05-06' '13:37:25.215","now_ts":"2024-05-06' '13:37:32.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_adapter_compatibility/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:37:34 CST 2024] <<<<<< run test case canal_json_adapter_compatibility success! >>>>>> /tmp/tidb_cdc_test/consistent_replicate_storage_file/tidb_other.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/tidb.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/pd1.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/sync_diff_inspector.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/tikv2.log /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_basic/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:37:37 CST 2024] <<<<<< run test case canal_json_basic success! >>>>>> /tmp/tidb_cdc_test/consistent_replicate_storage_file/tidb-slow.log =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_content_compatible/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:37:41 CST 2024] <<<<<< run test case canal_json_content_compatible success! >>>>>> + ls -alh log-G02.tar.gz -rw-r--r-- 1 jenkins jenkins 475M May 6 13:37 log-G02.tar.gz [Pipeline] archiveArtifacts Archiving artifacts Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_topics/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:37:44 CST 2024] <<<<<< run test case multi_topics success! >>>>>> Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319dea980003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:29939, start at 2024-05-06 13:36:41.126783373 +0800 CST m=+5.160671839 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:38:41.133 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:36:41.126 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:26:41.126 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319dea980003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:29939, start at 2024-05-06 13:36:41.126783373 +0800 CST m=+5.160671839 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:38:41.133 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:36:41.126 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:26:41.126 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319deb3c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:30030, start at 2024-05-06 13:36:41.170789089 +0800 CST m=+5.138271770 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:38:41.177 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:36:41.167 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:26:41.167 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/avro_basic/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:37:47 CST 2024] <<<<<< run test case avro_basic success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:37:50 CST 2024] <<<<<< run test case canal_json_handle_key_only success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:37:53 CST 2024] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_claim_check/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:37:56 CST 2024] <<<<<< run test case canal_json_claim_check success! >>>>>> Verifying downstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/open_protocol_claim_check/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:37:59 CST 2024] <<<<<< run test case open_protocol_claim_check success! >>>>>> Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_basic/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:38:03 CST 2024] <<<<<< run test case canal_json_storage_basic success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-80-g06ee59bd9c Edition: Community Git Commit Hash: 06ee59bd9c683757f75fdd3469f37f50988a1a2f Git Branch: master UTC Build Time: 2024-05-06 03:43:22 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319dea980003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:29939, start at 2024-05-06 13:36:41.126783373 +0800 CST m=+5.160671839 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:38:41.133 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:36:41.126 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:26:41.126 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319dea980003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:29939, start at 2024-05-06 13:36:41.126783373 +0800 CST m=+5.160671839 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:38:41.133 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:36:41.126 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:26:41.126 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d319deb3c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1779-1lcp8-4b2m9-z17sn, pid:30030, start at 2024-05-06 13:36:41.170789089 +0800 CST m=+5.138271770 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-13:38:41.177 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-13:36:41.167 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-13:26:41.167 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 3306 ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.35249.out cli tso query --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/run.sh using Sink-Type: mysql... <<================= [Mon May 6 13:38:06 CST 2024] <<<<<< run test case canal_json_storage_partition_table success! >>>>>> + set +x + tso='449570114450227201 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449570114450227201 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449570114450227201 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Mon May 6 13:38:07 CST 2024] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.3528735289.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_tables_ddl/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 05:38:10 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7365756350300007855 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:36:49.638926886 +0800 CST StartTs:449570093039616001 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001407b00 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570094350598146} {CheckpointTs:449570097601183752 MinTableBarrierTs:449570097863065607 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9eed3a71-1690-4211-86e7-a7a5d7739bd7 {"id":"9eed3a71-1690-4211-86e7-a7a5d7739bd7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973888} /tidb/cdc/default/__cdc_meta__/capture/d1c0ca7d-a018-45a3-8fc2-1afe545bec2d {"id":"d1c0ca7d-a018-45a3-8fc2-1afe545bec2d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973806} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c67548ad1 d1c0ca7d-a018-45a3-8fc2-1afe545bec2d /tidb/cdc/default/__cdc_meta__/owner/22318f4c689facd6 9eed3a71-1690-4211-86e7-a7a5d7739bd7 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7365756350300007855,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:36:49.638926886+08:00","start-ts":449570093039616001,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570094350598146} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":449570097601183752,"min-table-barrier-ts":449570097863065607,"admin-job-type":0} /tidb/cdc/default/default/task/position/9eed3a71-1690-4211-86e7-a7a5d7739bd7/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d1c0ca7d-a018-45a3-8fc2-1afe545bec2d/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365756350300007855 {"id":7365756350300007855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7365756350300007855 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:36:49.638926886 +0800 CST StartTs:449570093039616001 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001407b00 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570094350598146} {CheckpointTs:449570097601183752 MinTableBarrierTs:449570097863065607 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9eed3a71-1690-4211-86e7-a7a5d7739bd7 {"id":"9eed3a71-1690-4211-86e7-a7a5d7739bd7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973888} /tidb/cdc/default/__cdc_meta__/capture/d1c0ca7d-a018-45a3-8fc2-1afe545bec2d {"id":"d1c0ca7d-a018-45a3-8fc2-1afe545bec2d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973806} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c67548ad1 d1c0ca7d-a018-45a3-8fc2-1afe545bec2d /tidb/cdc/default/__cdc_meta__/owner/22318f4c689facd6 9eed3a71-1690-4211-86e7-a7a5d7739bd7 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7365756350300007855,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:36:49.638926886+08:00","start-ts":449570093039616001,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570094350598146} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":449570097601183752,"min-table-barrier-ts":449570097863065607,"admin-job-type":0} /tidb/cdc/default/default/task/position/9eed3a71-1690-4211-86e7-a7a5d7739bd7/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d1c0ca7d-a018-45a3-8fc2-1afe545bec2d/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc+ grep -q 'failed to get info:' /default/default/upstream/7365756350300007855 {"id":7365756350300007855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7365756350300007855 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 13:36:49.638926886 +0800 CST StartTs:449570093039616001 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001407b00 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-gd72d260c0 Epoch:449570094350598146} {CheckpointTs:449570097601183752 MinTableBarrierTs:449570097863065607 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9eed3a71-1690-4211-86e7-a7a5d7739bd7 {"id":"9eed3a71-1690-4211-86e7-a7a5d7739bd7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973888} /tidb/cdc/default/__cdc_meta__/capture/d1c0ca7d-a018-45a3-8fc2-1afe545bec2d {"id":"d1c0ca7d-a018-45a3-8fc2-1afe545bec2d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-gd72d260c0","git-hash":"d72d260c017170db94deda602246fcddfea2a4a1","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714973806} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4c67548ad1 d1c0ca7d-a018-45a3-8fc2-1afe545bec2d /tidb/cdc/default/__cdc_meta__/owner/22318f4c689facd6 9eed3a71-1690-4211-86e7-a7a5d7739bd7 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7365756350300007855,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T13:36:49.638926886+08:00","start-ts":449570093039616001,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-gd72d260c0","epoch":449570094350598146} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":449570097601183752,"min-table-barrier-ts":449570097863065607,"admin-job-type":0} /tidb/cdc/default/default/task/position/9eed3a71-1690-4211-86e7-a7a5d7739bd7/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d1c0ca7d-a018-45a3-8fc2-1afe545bec2d/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365756350300007855 {"id":7365756350300007855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449570114450227201 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.35345.out cli changefeed create --start-ts=449570114450227201 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCCFAAF0F6CBA7 < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 05:38:11 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... Create changefeed successfully! ID: test-2 Info: {"upstream_id":7365756350300007855,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T13:38:12.251713128+08:00","start_ts":449570114450227201,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-gd72d260c0","resolved_ts":449570114450227201,"checkpoint_ts":449570114450227201,"checkpoint_time":"2024-05-06 13:38:06.300"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Recording fingerprints + set +x + run_sql 'use test; create table t2(id bigint primary key, a date); insert into t2 values(1, '\''2023-02-08'\'');' 127.0.0.1 4000 start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G02' Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes Exiting on signal: TERMINATED script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 [Pipeline] } Cache not saved (inner-step execution failed) ++ stop_tidb_cluster script returned exit code 143 [Pipeline] // cache [Pipeline] } [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // dir [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] // withCredentials [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] // withCredentials [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] // container [Pipeline] } [Pipeline] } ++ stop_tidb_cluster script returned exit code 143 [Pipeline] // node [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // podTemplate [Pipeline] // node [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] // podTemplate [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // withEnv [Pipeline] // withCredentials [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G01' [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // timeout [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G10' [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G09' ++ stop_tidb_cluster /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh: line 1: 16731 Terminated sleep 130 script returned exit code 143 [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G08' [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline ERROR: script returned exit code 1 Finished: FAILURE