Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/latest/pull_cdc_integration_pulsar_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-xzk9-2jt3c Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-xzk9-2jt3c is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-xzk9j-544mh --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "3cd5d079b9f128f3b1b02fc850e2277cb8e92943" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-xzk9j" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-xzk9-2jt3c" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-xzk9-2jt3c" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-xzk9-2jt3c" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-xzk9-2jt3c in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-list --no-walk 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=236b19fb-a619-4e8a-8b08-61ec4825a770 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-xzk9-2jt3c HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-xzk9j GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-100f7bb7add7cfb003dff5768bf20348b91a6b6cef38154d1625c661ef800a11 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-xzk9-2jt3c pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-xzk9j GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-xzk9-2jt3c GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.6' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build2815771=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-xzk9-2jt3c bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-xzk9-2jt3c bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 477 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: 3ef4ed0eb116a412 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 26 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Mon May 06 08:41:18 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11023/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11023/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11023/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11023/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [scripts/download-integration-test-binaries.sh, tests/integration_tests/sequence/run.sh] [Pipeline] echo diff file not matched: scripts/download-integration-test-binaries.sh [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-be15534-e9f061d) 203651072 bytes in 1.66 secs (122908598 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/.git/ .git HEAD is now at e9f061d7a enable sequence test POST git-upload-pack (656 bytes) From https://github.com/pingcap/tiflow = [up to date] master -> origin/master = [up to date] refs/pull/11023/head -> origin/pr/11023/head Previous HEAD position was e9f061d7a enable sequence test HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) ๐Ÿšง Checkouting to base SHA:be1553484fe4c03594eabb8d7435c694e5fd7224... HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: be1553484fe4c03594eabb8d7435c694e5fd7224 be1553484 codec(ticdc): avro simplify the unit test (#11010) 2a7a65c6f Support Sequences (#10203) 36e9e1bf6 cli(ticdc): allow client authentication to be enabled without tls (#11005) ๐Ÿšง Pre-merge heads of pull requests to base SHA: be1553484fe4c03594eabb8d7435c694e5fd7224 ... Updating be1553484..e9f061d7a Fast-forward scripts/download-integration-test-binaries.sh | 4 ++-- tests/integration_tests/sequence/run.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) ๐Ÿงพ Pre-merged result: e9f061d7a0898b066e3c6e271294672cae36f6d6 e9f061d7a enable sequence test be1553484 codec(ticdc): avro simplify the unit test (#11010) 2a7a65c6f Support Sequences (#10203) โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-be15534-e9f061d already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./scripts/download-integration-test-binaries.sh master Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 651 0 --:--:-- --:--:-- --:--:-- 661 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1598 0 --:--:-- --:--:-- --:--:-- 1640 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 901 0 --:--:-- --:--:-- --:--:-- 911 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 625 0 --:--:-- --:--:-- --:--:-- 630 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/659f32a81300f9dbcea9032b3c8e4825555ccfd1/centos7/tidb-server.tar.gz 2024-05-06 16:41:44 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/659f32a81300f9dbcea9032b3c8e4825555ccfd1/centos7/tidb-server.tar.gz [536598984/536598984] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz 2024-05-06 16:41:59 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz [187372022/187372022] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/88099c95a3c0c13a827c0795c3d45070264665e4/centos7/tikv-server.tar.gz 2024-05-06 16:42:22 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/88099c95a3c0c13a827c0795c3d45070264665e4/centos7/tikv-server.tar.gz [919298452/919298452] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e50de84e6d6ecdcc108990217b70b6bb3f50271/centos7/tiflash.tar.gz 2024-05-06 16:42:52 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e50de84e6d6ecdcc108990217b70b6bb3f50271/centos7/tiflash.tar.gz [456051121/456051121] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-06 16:42:57 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-05-06 16:43:01 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-05-06 16:43:01 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-05-06 16:43:02 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-79f1fd1e_linux-amd64.tar.gz 2024-05-06 16:43:05 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-79f1fd1e_linux-amd64.tar.gz [55896256/55896256] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> download schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2024-05-06 16:43:19 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 6 16:43 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 16:43 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 6 16:43 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 6 16:43 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 6 12:44 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 6 12:44 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 6 12:44 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 6 12:44 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 6 12:05 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M May 6 12:45 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x. 1 jenkins jenkins 133M May 6 16:40 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 6 15:59 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M May 6 12:44 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M May 6 15:33 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/pd-ut ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ../tiflow/bin/xprog ./bin/ + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 6 16:43 . drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 16:43 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 6 16:43 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 6 16:43 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 6 12:44 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 6 12:44 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 6 12:44 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 6 12:44 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 6 12:05 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M May 6 12:45 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x. 1 jenkins jenkins 133M May 6 16:40 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 6 15:59 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M May 6 12:44 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M May 6 15:33 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + ./bin/tidb-server -V Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 + ./bin/tikv-server -V TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.1.5-2-geae7b5d Git Commit Hash: eae7b5d149e30a02a04ec6116e08b0a8f07fe4aa Git Branch: disable-cgo-for-sync-diff UTC Build Time: 2024-05-06 08:40:40 Go Version: go1.22.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x. 2 jenkins jenkins 4.0K May 6 16:43 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 16:43 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-ge9f061d7a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 08:43:22" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e9f061d7a0898b066e3c6e271294672cae36f6d6" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-ge9f061d7a"' -o bin/cdc ./cmd/cdc go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading golang.org/x/sync v0.7.0 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/google/uuid v1.6.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/YangKeao/seahash v0.0.0-20240229041150-e7bf269c3140 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading golang.org/x/sys v0.19.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/goccy/go-json v0.10.2 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/swaggo/swag v1.16.3 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/golang/protobuf v1.5.4 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading golang.org/x/text v0.14.0 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/go-openapi/spec v0.21.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading github.com/glebarez/sqlite v1.7.0 go: downloading gorm.io/driver/mysql v1.3.3 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/glebarez/go-sqlite v1.21.2 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/go-openapi/jsonpointer v0.21.0 go: downloading github.com/go-openapi/jsonreference v0.21.0 go: downloading github.com/go-openapi/swag v0.23.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/kr/pretty v0.3.1 go: downloading modernc.org/libc v1.37.1 go: downloading modernc.org/sqlite v1.27.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/apache/thrift v0.16.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading k8s.io/apimachinery v0.28.6 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading modernc.org/memory v1.7.2 go: downloading modernc.org/mathutil v1.6.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + '[' -f ./bin/cdc_pulsar_consumer ']' + make pulsar_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-ge9f061d7a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 08:45:24" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e9f061d7a0898b066e3c6e271294672cae36f6d6" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-ge9f061d7a"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-ge9f061d7a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 08:45:31" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e9f061d7a0898b066e3c6e271294672cae36f6d6" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-ge9f061d7a"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-ge9f061d7a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 08:45:31" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e9f061d7a0898b066e3c6e271294672cae36f6d6" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-ge9f061d7a"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-ge9f061d7a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 08:45:31" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e9f061d7a0898b066e3c6e271294672cae36f6d6" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-ge9f061d7a"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-ge9f061d7a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 08:45:31" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e9f061d7a0898b066e3c6e271294672cae36f6d6" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-ge9f061d7a"' -o bin/oauth2-server ./cmd/oauth2-server/main.go go: downloading github.com/go-oauth2/oauth2/v4 v4.5.2 go: downloading github.com/tidwall/buntdb v1.3.0 go: downloading github.com/tidwall/rtred v0.1.2 go: downloading github.com/tidwall/grect v0.1.4 go: downloading github.com/tidwall/match v1.1.1 go: downloading github.com/tidwall/gjson v1.14.3 go: downloading github.com/tidwall/tinyqueue v0.1.1 go: downloading github.com/tidwall/pretty v1.2.0 $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20231213065948-336e064b488d go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading go.uber.org/dig v1.13.0 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading go.uber.org/goleak v1.3.0 go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-ge9f061d7a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 08:45:31" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e9f061d7a0898b066e3c6e271294672cae36f6d6" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-ge9f061d7a"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-ge9f061d7a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 08:45:31" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e9f061d7a0898b066e3c6e271294672cae36f6d6" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-ge9f061d7a"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.2G drwxr-sr-x. 2 jenkins jenkins 4.0K May 6 16:50 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 16:43 .. -rwxr-xr-x. 1 jenkins jenkins 220M May 6 16:50 cdc -rwxr-xr-x. 1 jenkins jenkins 363M May 6 16:49 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 6 16:45 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 183M May 6 16:45 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 6 16:45 cdc_storage_consumer -rwxr-xr-x. 1 jenkins jenkins 12M May 6 16:45 oauth2-server + ./bin/cdc version Release Version: v8.2.0-alpha-22-ge9f061d7a Git Commit Hash: e9f061d7a0898b066e3c6e271294672cae36f6d6 Git Branch: HEAD UTC Build Time: 2024-05-06 08:45:31 Go Version: go version go1.21.6 linux/amd64 Failpoint Build: true [Pipeline] } Cache saved successfully (binary/pingcap/tiflow/cdc-integration-pulsar-test/rev-be15534-e9f061d) 1194820608 bytes in 28.23 secs (42317656 bytes/sec) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 3.0G drwxr-sr-x. 6 jenkins jenkins 4.0K May 6 16:50 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 16:43 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 6 16:50 bin -rwxr-xr-x. 1 jenkins jenkins 220M May 6 16:50 cdc -rwxr-xr-x. 1 jenkins jenkins 363M May 6 16:49 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 6 16:45 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 183M May 6 16:45 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 6 16:45 cdc_storage_consumer drwxr-sr-x. 4 jenkins jenkins 4.0K May 6 16:50 etc -rwxr-xr-x. 1 jenkins jenkins 17M May 6 16:50 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 6 16:50 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 6 16:50 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 16:50 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 6 16:50 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K May 6 16:50 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 6 16:50 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K May 6 16:50 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 6 16:50 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 6 16:50 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 6 16:50 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M May 6 16:50 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M May 6 16:50 minio -rwxr-xr-x. 1 jenkins jenkins 12M May 6 16:45 oauth2-server -rwxr-xr-x. 1 jenkins jenkins 37M May 6 16:50 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M May 6 16:50 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M May 6 16:50 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M May 6 16:50 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M May 6 16:50 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M May 6 16:50 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M May 6 16:50 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M May 6 16:50 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 6 16:50 share -rwxr-xr-x. 1 jenkins jenkins 32M May 6 16:50 stores-dump -rwxr-xr-x. 1 jenkins jenkins 133M May 6 16:50 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 6 16:50 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M May 6 16:50 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M May 6 16:50 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M May 6 16:50 xprog [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 64.09 secs (57234578 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-m6vg-ft0kp Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-jgqn-92trb [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq [Pipeline] podTemplate Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-m6vg-ft0kp is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-m6vgh-tlvwc --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "857428c4b4522d0cbf031886bddc31feac75e36f" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-m6vgh" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-m6vg-ft0kp" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-m6vg-ft0kp" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-m6vg-ft0kp" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" [Pipeline] { Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-m6vg-ft0kp in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test [Pipeline] { [Pipeline] node [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-g720-bjdgs No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7eb893a9; decorates RemoteLauncher[hudson.remoting.Channel@670ee89e:JNLP4-connect connection from 10.233.70.184/10.233.70.184:36786] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-h5k3-ghhrj Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-jgqn-92trb is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-jgqnd-q76wq --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "954a6a0ee2d65fa1512975cdee25e2b979970080" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-jgqnd" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-jgqn-92trb" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-jgqn-92trb" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-jgqn-92trb" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-jgqn-92trb in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0g2h-bv6sm No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1d7c7e1e; decorates RemoteLauncher[hudson.remoting.Channel@7494621d:JNLP4-connect connection from 10.233.67.56/10.233.67.56:46258] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-g720-bjdgs is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-g720n-6xsb2 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "5659bb73f4bf821cb9da61a4b673fc4091ff230e" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-g720n" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-g720-bjdgs" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-g720-bjdgs" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-g720-bjdgs" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" [Pipeline] cache Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-g720-bjdgs in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-h5k3-ghhrj is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-h5k38-0xczb --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a1335f67905635268fc2f09c8553ece4796b00db" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-h5k38" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-h5k3-ghhrj" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-h5k3-ghhrj" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-h5k3-ghhrj" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-h5k3-ghhrj in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-4q3d-6b3lc Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0g2h-bv6sm is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-0g2hz-n4mjv --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "1fd27fa609e2120e096f4a5ea8ada2f64757ea27" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-0g2hz" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0g2h-bv6sm" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0g2h-bv6sm" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0g2h-bv6sm" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0g2h-bv6sm in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-9btdz-s2rhf --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "0671aad411bf0908bee326f822672b1e26074e4c" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-9btdz" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-4q3d-6b3lc is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-4q3dm-pbddq --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a1ec76e7ed749c1da4c715c081e3c71ac4968a66" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-4q3dm" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-4q3d-6b3lc" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-4q3d-6b3lc" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-4q3d-6b3lc" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-4q3d-6b3lc in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-04884-mx7f5 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "99e65b5075f5584eda28e5a1c85cf0485dab51f2" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-04884" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 15.33 secs (239265797 bytes/sec) [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] { [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G00 Run cases: bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_suicide server_config_compatibility kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro debezium lossy_ddl storage_csv_update PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=2bf519f0-6a6e-416c-8d4d-327bb3583bb8 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-m6vg-ft0kp HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-m6vgh GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-m6vgh pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-m6vg-ft0kp GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-m6vg-ft0kp GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:52:23 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] { The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] container The recommended git tool is: git [Pipeline] { [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] stage The recommended git tool is: git [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] cache Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-j6m9-tmj32 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-q6kw-vxk5b Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7txd-3l8p2 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0c9z-7hn20 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-x5q3-r7q9p Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-19s0-56793 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-5fm7-zn46k Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-1bd3-dwfrg Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-rjrh-lnz8j No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3018b5e8; decorates RemoteLauncher[hudson.remoting.Channel@55ec5f1:JNLP4-connect connection from 10.233.90.157/10.233.90.157:35110] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6198af30; decorates RemoteLauncher[hudson.remoting.Channel@48bd17db:JNLP4-connect connection from 10.233.108.150/10.233.108.150:45958] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6f8dbd6b; decorates RemoteLauncher[hudson.remoting.Channel@27b19fc5:JNLP4-connect connection from 10.233.107.46/10.233.107.46:39152] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@62d1dbae; decorates RemoteLauncher[hudson.remoting.Channel@4c03433e:JNLP4-connect connection from 10.233.106.42/10.233.106.42:38772] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@77327e63; decorates RemoteLauncher[hudson.remoting.Channel@48ba519d:JNLP4-connect connection from 10.233.69.55/10.233.69.55:55948] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2d242195; decorates RemoteLauncher[hudson.remoting.Channel@67f28d26:JNLP4-connect connection from 10.233.97.170/10.233.97.170:39890] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:52:26 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-q6kw-vxk5b is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-q6kwb-0bw2m --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d2465f134feae4567eb5a5d21b49319192970d79" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-q6kwb" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-q6kw-vxk5b" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-q6kw-vxk5b" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-q6kw-vxk5b" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-q6kw-vxk5b in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-rjrh-lnz8j is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-rjrhn-ftk0s --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8fca5c6cfe3a2bd82af9064efe4b1b7398503af3" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-rjrhn" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-rjrh-lnz8j" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-rjrh-lnz8j" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-rjrh-lnz8j" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-rjrh-lnz8j in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-19s0-56793 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-19s0g-xrdlp --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "e7a3fcada30f4a696b04548548fbce14964d4163" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-19s0g" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-19s0-56793" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-19s0-56793" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-19s0-56793" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-19s0-56793 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7txd-3l8p2 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-7txd9-hcgzg --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "0159b4e26f95994f551480a5ca53df4d715008f0" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-7txd9" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7txd-3l8p2" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7txd-3l8p2" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7txd-3l8p2" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7txd-3l8p2 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-j6m9-tmj32 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-j6m97-psp1q --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "adfb851bace2de51bb5316f64f71175557027fd7" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-j6m97" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-j6m9-tmj32" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-j6m9-tmj32" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-j6m9-tmj32" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-j6m9-tmj32 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0c9z-7hn20 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-0c9zg-g1g1n --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "cee558a3ca6b2d29d0f26ae12bd43e6b7e9d5ab5" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-0c9zg" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0c9z-7hn20" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0c9z-7hn20" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0c9z-7hn20" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0c9z-7hn20 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: pulsar... <<================= kafka downstream isn't support syncpoint record [Mon May 6 16:52:29 CST 2024] <<<<<< run test case syncpoint success! >>>>>> > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-x5q3-r7q9p is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-x5q3s-9qpb9 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c0141364734b3a42f48963a815f570b4bdb47437" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-x5q3s" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-x5q3-r7q9p" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-x5q3-r7q9p" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-x5q3-r7q9p" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-x5q3-r7q9p in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-1bd3-dwfrg is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-1bd30-z3ps2 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "6e72f13146305615069b15a45b67b92d64835c4d" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-1bd30" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-1bd3-dwfrg" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-1bd3-dwfrg" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-1bd3-dwfrg" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-1bd3-dwfrg in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-5fm7-zn46k is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-5fm70-8sqkt --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "bb68d47ac32533008f538ae0245cd90f907f2d28" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-5fm70" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-5fm7-zn46k" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-5fm7-zn46k" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-5fm7-zn46k" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-5fm7-zn46k in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7wx1-4ndrn find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:52:32 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:52:35 CST 2024] <<<<<< run test case server_config_compatibility success! >>>>>> Still waiting to schedule task โ€˜pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7wx1-4ndrnโ€™ is offline find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:52:39 CST 2024] <<<<<< run test case kafka_big_messages success! >>>>>> Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7wx1-4ndrn is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-7wx1w-59dd8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "fbe5e82dabc38b2f1a11b6f650bc99f72399827a" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-7wx1w" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7wx1-4ndrn" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7wx1-4ndrn" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7wx1-4ndrn" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7wx1-4ndrn in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:52:42 CST 2024] <<<<<< run test case kafka_compression success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 17.81 secs (205894955 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=220c498d-4c76-4cad-a2aa-e1daaada8275 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-jgqn-92trb HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-jgqnd GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-jgqnd pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-jgqn-92trb GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-jgqn-92trb GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:52:45 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] { [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:52:45 CST 2024] <<<<<< run test case kafka_messages success! >>>>>> [Pipeline] checkout The recommended git tool is: git [Pipeline] container [Pipeline] { [Pipeline] container The recommended git tool is: git [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@57c60705; decorates RemoteLauncher[hudson.remoting.Channel@62411e8f:JNLP4-connect connection from 10.233.123.25/10.233.123.25:36042] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2ef3d922; decorates RemoteLauncher[hudson.remoting.Channel@7bca921a:JNLP4-connect connection from 10.233.127.187/10.233.127.187:47588] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] cache Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@77d1286d; decorates RemoteLauncher[hudson.remoting.Channel@2d2c06f5:JNLP4-connect connection from 10.233.105.25/10.233.105.25:39966] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7c8e5d53; decorates RemoteLauncher[hudson.remoting.Channel@705cfe45:JNLP4-connect connection from 10.233.100.251/10.233.100.251:39482] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3a41c39a; decorates RemoteLauncher[hudson.remoting.Channel@39b72f2b:JNLP4-connect connection from 10.233.86.48/10.233.86.48:42790] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3178b207; decorates RemoteLauncher[hudson.remoting.Channel@397de40f:JNLP4-connect connection from 10.233.126.32/10.233.126.32:45248] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3c5069c2; decorates RemoteLauncher[hudson.remoting.Channel@8ce4ebc:JNLP4-connect connection from 10.233.66.35/10.233.66.35:46600] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@11ff5f2e; decorates RemoteLauncher[hudson.remoting.Channel@7b91d3e2:JNLP4-connect connection from 10.233.71.114/10.233.71.114:48036] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@55f79d74; decorates RemoteLauncher[hudson.remoting.Channel@1009ba02:JNLP4-connect connection from 10.233.72.149/10.233.72.149:34158] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@638b70a0; decorates RemoteLauncher[hudson.remoting.Channel@3ecb342b:JNLP4-connect connection from 10.233.68.228/10.233.68.228:37558] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:52:48 CST 2024] <<<<<< run test case kafka_sink_error_resume success! >>>>>> > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: pulsar... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCDA4A9426D873 < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 08:52:51 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Mon May 6 16:52:52 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:52:52 CST 2024] <<<<<< run test case mq_sink_lost_callback success! >>>>>> Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:52:55 CST 2024] <<<<<< run test case mq_sink_dispatcher success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:52:56 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 8.17 secs (448926284 bytes/sec) [Pipeline] { [Pipeline] cache find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:52:58 CST 2024] <<<<<< run test case kafka_column_selector success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:53:00 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_column_selector_avro/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:53:01 CST 2024] <<<<<< run test case kafka_column_selector_avro success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:53:04 CST 2024] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/debezium/run.sh using Sink-Type: pulsar... <<================= find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:53:05 CST 2024] <<<<<< run test case lossy_ddl success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: pulsar... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: pulsar... <<================= You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend [Mon May 6 16:53:08 CST 2024] <<<<<< run test case storage_csv_update success! >>>>>> Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCDA4EBB165BFA < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 08:53:09 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Mon May 6 16:53:09 CST 2024] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Exiting on signal: INTERRUPT \033[0;36m<<< Run all test success >>>\033[0m =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:53:13 CST 2024] <<<<<< run test case consistent_partition_table success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 16.89 secs (217141605 bytes/sec) [Pipeline] { [Pipeline] cache find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_big_messages_v2/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:53:17 CST 2024] <<<<<< run test case kafka_big_messages_v2 success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_tables_ddl_v2/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:53:20 CST 2024] <<<<<< run test case multi_tables_ddl_v2 success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:53:24 CST 2024] <<<<<< run test case multi_topics_v2 success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: pulsar... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup + CDC_BINARY=cdc.test + SINK_TYPE=pulsar + EXIST_FILES=() + CLEANED_FILES=() + trap stop_tidb_cluster EXIT + run pulsar + '[' pulsar '!=' storage ']' + return + check_logs /tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Mon May 6 16:53:27 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>>' [Mon May 6 16:53:27 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>> + stop_tidb_cluster find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:53:31 CST 2024] <<<<<< run test case csv_storage_basic success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:53:34 CST 2024] <<<<<< run test case csv_storage_multi_tables_ddl success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/csv_storage_partition_table/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:53:38 CST 2024] <<<<<< run test case csv_storage_partition_table success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 24.16 secs (151841360 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 7.76 secs (472912237 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 19.98 secs (183597126 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 22.46 secs (163279556 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G04 Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=92a7a19f-7212-4a0a-a0a2-93a44a4aebc7 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-g720-bjdgs HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-g720n GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-g720-bjdgs pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-g720n GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-g720-bjdgs GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G05 Run cases: charset_gbk ddl_manager multi_source PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=9c58a288-b21f-4eac-814c-bc0c36eda34f BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-h5k3-ghhrj HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-h5k38 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-h5k3-ghhrj pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-h5k38 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-h5k3-ghhrj GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G01 Run cases: http_api http_api_tls api_v2 http_api_tls_with_user_auth cli_tls_with_auth kafka_simple_basic kafka_simple_basic_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_simple_claim_check kafka_simple_claim_check_avro canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible multi_topics avro_basic canal_json_handle_key_only open_protocol_handle_key_only canal_json_claim_check open_protocol_claim_check canal_json_storage_basic canal_json_storage_partition_table multi_tables_ddl PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=0be42eae-ddda-4337-aca6-616de2415435 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0g2h-bv6sm HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-0g2hz GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0g2h-bv6sm pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-0g2hz GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0g2h-bv6sm GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:54:42 CST 2024] <<<<<< run test case http_api success! >>>>>> [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=802f45b4-b573-48a8-8e95-67a7cf737ec7 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-4q3d-6b3lc HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-4q3dm GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-4q3d-6b3lc pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-4q3dm GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-4q3d-6b3lc GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G03 Run cases: row_format drop_many_tables processor_stop_delay partition_table PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=3b7d3b27-2b69-4c50-8b4a-c9a6ad81d74b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-04884 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-04884 pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc already exists) [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc already exists) [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=30c6aaf0-a4b2-4fd2-8c9f-a2acac1868eb BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-9btdz GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-9btdz pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] // cache [Pipeline] } [Pipeline] // cache [Pipeline] } [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] // dir find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/http_api_tls/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:54:46 CST 2024] <<<<<< run test case http_api_tls success! >>>>>> [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... [Pipeline] cache Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/api_v2/run.sh using Sink-Type: pulsar... <<================= Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/http_api_tls_with_user_auth/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:54:54 CST 2024] <<<<<< run test case http_api_tls_with_user_auth success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/cli_tls_with_auth/run.sh using Sink-Type: pulsar... <<================= ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34740ac00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-4q3d-6b3lc, pid:1362, start at 2024-05-06 16:54:57.989964392 +0800 CST m=+5.129265741 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:56:57.999 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:54:57.968 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:44:57.968 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34740ac00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-4q3d-6b3lc, pid:1362, start at 2024-05-06 16:54:57.989964392 +0800 CST m=+5.129265741 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:56:57.999 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:54:57.968 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:44:57.968 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34740dd00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-4q3d-6b3lc, pid:1441, start at 2024-05-06 16:54:58.170919005 +0800 CST m=+5.259714093 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:56:58.178 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:54:58.164 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:44:58.164 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347425740005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8, pid:1357, start at 2024-05-06 16:54:59.682339751 +0800 CST m=+5.412625923 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:56:59.692 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:54:59.677 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:44:59.677 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34741f240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq, pid:1334, start at 2024-05-06 16:54:59.319351561 +0800 CST m=+6.905635374 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:56:59.337 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:54:59.322 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:44:59.322 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34741f240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq, pid:1334, start at 2024-05-06 16:54:59.319351561 +0800 CST m=+6.905635374 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:56:59.337 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:54:59.322 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:44:59.322 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3474214c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq, pid:1388, start at 2024-05-06 16:54:59.432016268 +0800 CST m=+6.938629789 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:56:59.442 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:54:59.411 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:44:59.411 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347409d4001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-h5k3-ghhrj, pid:1430, start at 2024-05-06 16:54:57.959888298 +0800 CST m=+6.321138919 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:56:57.966 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:54:57.959 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:44:57.959 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347409d4001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-h5k3-ghhrj, pid:1430, start at 2024-05-06 16:54:57.959888298 +0800 CST m=+6.321138919 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:56:57.966 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:54:57.959 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:44:57.959 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3474082c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-h5k3-ghhrj, pid:1494, start at 2024-05-06 16:54:57.847673772 +0800 CST m=+5.097386817 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:56:57.854 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:54:57.853 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:44:57.853 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ++ date + echo '[Mon May 6 16:55:01 CST 2024] <<<<<< START pulsar cluster in kv_client_stream_reconnect case >>>>>>' [Mon May 6 16:55:01 CST 2024] <<<<<< START pulsar cluster in kv_client_stream_reconnect case >>>>>> + workdir=/tmp/tidb_cdc_test/kv_client_stream_reconnect + cluster_type=oauth + cd /tmp/tidb_cdc_test/kv_client_stream_reconnect + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/kv_client_stream_reconnect/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/kv_client_stream_reconnect/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/kv_client_stream_reconnect/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/kv_client_stream_reconnect/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/kv_client_stream_reconnect/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/kv_client_stream_reconnect/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/kv_client_stream_reconnect/pulsar_standalone.conf + pulsar_port=6650 + '[' oauth == mtls ']' + '[' oauth == oauth ']' + echo 'Waiting for oauth2 server to be ready...' Waiting for oauth2 server to be ready... + i=0 + nc -z localhost 9096 + oauth2-server + gen_oauth_config + echo ' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/kv_client_stream_reconnect/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/kv_client_stream_reconnect/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ date + echo '[Mon May 6 16:55:01 CST 2024] <<<<<< START pulsar cluster in oauth mode in kv_client_stream_reconnect case >>>>>>' [Mon May 6 16:55:01 CST 2024] <<<<<< START pulsar cluster in oauth mode in kv_client_stream_reconnect case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/kv_client_stream_reconnect/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347425740005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8, pid:1357, start at 2024-05-06 16:54:59.682339751 +0800 CST m=+5.412625923 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:56:59.692 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:54:59.677 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:44:59.677 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3474263c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8, pid:1447, start at 2024-05-06 16:54:59.775713363 +0800 CST m=+5.438988535 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:56:59.782 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:54:59.777 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:44:59.777 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2676.out cli tso query --pd=http://127.0.0.1:2379 [Mon May 6 16:55:03 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.28762878.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34744d28000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-g720-bjdgs, pid:1411, start at 2024-05-06 16:55:02.232518163 +0800 CST m=+6.499073793 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:57:02.241 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:55:02.218 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:45:02.218 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34744d28000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-g720-bjdgs, pid:1411, start at 2024-05-06 16:55:02.232518163 +0800 CST m=+6.499073793 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:57:02.241 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:55:02.218 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:45:02.218 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34743d840017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-g720-bjdgs, pid:1501, start at 2024-05-06 16:55:01.261904606 +0800 CST m=+5.467333839 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:57:01.271 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:55:01.268 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:45:01.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2828.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449573212251226113 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573212251226113 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:55:05 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.27052707.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2848.out cli tso query --pd=http://127.0.0.1:2379 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_simple_basic/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:55:05 CST 2024] <<<<<< run test case kafka_simple_basic success! >>>>>> + set +x + tso='449573212632645633 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573212632645633 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "recordcount"="10" "insertproportion"="0" "mysql.user"="root" "dotransactions"="false" "workload"="core" "operationcount"="0" "mysql.port"="4000" "threadcount"="2" "updateproportion"="0" "requestdistribution"="uniform" "readallfields"="true" "readproportion"="0" "mysql.db"="sink_retry" "mysql.host"="127.0.0.1" "scanproportion"="0" ********************************************** Run finished, takes 10.340253ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1740.0, Avg(us): 1995, Min(us): 1264, Max(us): 4621, 95th(us): 5000, 99th(us): 5000 [Mon May 6 16:55:06 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.28902892.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:55:06 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d41eee8-b3ce-499d-9331-28183c99004d {"id":"2d41eee8-b3ce-499d-9331-28183c99004d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985703} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1cd3f3d1 2d41eee8-b3ce-499d-9331-28183c99004d /tidb/cdc/default/default/upstream/7365807435054545416 {"id":7365807435054545416,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d41eee8-b3ce-499d-9331-28183c99004d {"id":"2d41eee8-b3ce-499d-9331-28183c99004d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985703} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1cd3f3d1 2d41eee8-b3ce-499d-9331-28183c99004d /tidb/cdc/default/default/upstream/7365807435054545416 {"id":7365807435054545416,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d41eee8-b3ce-499d-9331-28183c99004d {"id":"2d41eee8-b3ce-499d-9331-28183c99004d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985703} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1cd3f3d1 2d41eee8-b3ce-499d-9331-28183c99004d /tidb/cdc/default/default/upstream/7365807435054545416 {"id":7365807435054545416,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:55:06 CST 2024] <<<<<< START pulsar cluster in charset_gbk case >>>>>>' [Mon May 6 16:55:06 CST 2024] <<<<<< START pulsar cluster in charset_gbk case >>>>>> + workdir=/tmp/tidb_cdc_test/charset_gbk + cluster_type=mtls + cd /tmp/tidb_cdc_test/charset_gbk + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/charset_gbk/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/charset_gbk/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/charset_gbk/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/charset_gbk/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/charset_gbk/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/charset_gbk/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/charset_gbk/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/charset_gbk/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/charset_gbk/pulsar_standalone.conf + pulsar_port=6650 + '[' mtls == mtls ']' + pulsar_port=6651 + gen_mtls_config + openssl genrsa -out ca.key.pem 2048 Generating RSA private key, 2048 bit long modulus ...........+++ .............................................................................................................................................................+++ e is 65537 (0x10001) + openssl req -x509 -new -nodes -key ca.key.pem -subj /CN=CARoot -days 365 -out ca.cert.pem + openssl genrsa -out server.key.pem 2048 Generating RSA private key, 2048 bit long modulus ....................................+++ .+++ e is 65537 (0x10001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in server.key.pem -out server.key-pk8.pem -nocrypt + echo '[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + openssl req -new -config server.conf -key server.key.pem -out server.csr.pem -sha256 + openssl x509 -req -in server.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out server.cert.pem -days 365 -extensions v3_ext -extfile server.conf -sha256 Signature ok subject=/CN=server Getting CA Private Key + openssl genrsa -out broker_client.key.pem 2048 Generating RSA private key, 2048 bit long modulus .............................................................................................................................................................+++ ..+++ e is 65537 (0x10001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in broker_client.key.pem -out broker_client.key-pk8.pem -nocrypt + openssl req -new -subj /CN=broker_client -key broker_client.key.pem -out broker_client.csr.pem -sha256 + openssl x509 -req -in broker_client.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out broker_client.cert.pem -days 365 -sha256 Signature ok subject=/CN=broker_client Getting CA Private Key + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/charset_gbk/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/charset_gbk/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/charset_gbk/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/charset_gbk/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' + echo ' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/charset_gbk/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/charset_gbk/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem' ++ date + echo '[Mon May 6 16:55:06 CST 2024] <<<<<< START pulsar cluster in mtls mode in charset_gbk case >>>>>>' [Mon May 6 16:55:06 CST 2024] <<<<<< START pulsar cluster in mtls mode in charset_gbk case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6651 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/charset_gbk/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/charset_gbk/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/charset_gbk/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 public/default ++ date + echo '[Mon May 6 16:55:07 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:55:07 CST 2024] <<<<<< pulsar is ready >>>>>> [Mon May 6 16:55:07 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.32713273.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449573213036609538 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573213036609538 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:55:07 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.28902892.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + nc -z localhost 6651 + i=2 + '[' 2 -gt 20 ']' + sleep 2 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:55:08 CST 2024] <<<<<< run test case kafka_simple_basic_avro success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:55:08 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cdd0f041-bb08-4b55-a8df-005971f1874e {"id":"cdd0f041-bb08-4b55-a8df-005971f1874e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985705} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1cdcc3d4 cdd0f041-bb08-4b55-a8df-005971f1874e /tidb/cdc/default/default/upstream/7365807449580835332 {"id":7365807449580835332,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cdd0f041-bb08-4b55-a8df-005971f1874e {"id":"cdd0f041-bb08-4b55-a8df-005971f1874e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985705} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1cdcc3d4 cdd0f041-bb08-4b55-a8df-005971f1874e /tidb/cdc/default/default/upstream/7365807449580835332 {"id":7365807449580835332,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cdd0f041-bb08-4b55-a8df-005971f1874e {"id":"cdd0f041-bb08-4b55-a8df-005971f1874e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985705} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1cdcc3d4 cdd0f041-bb08-4b55-a8df-005971f1874e /tidb/cdc/default/default/upstream/7365807449580835332 {"id":7365807449580835332,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:55:08 CST 2024] <<<<<< START pulsar cluster in row_format case >>>>>>' [Mon May 6 16:55:08 CST 2024] <<<<<< START pulsar cluster in row_format case >>>>>> + workdir=/tmp/tidb_cdc_test/row_format + cluster_type=normal + cd /tmp/tidb_cdc_test/row_format + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/row_format/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/row_format/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/row_format/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/row_format/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/row_format/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/row_format/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/row_format/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/row_format/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/row_format/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/row_format/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/row_format/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/row_format/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:55:08 CST 2024] <<<<<< START pulsar cluster in normal mode in row_format case >>>>>>' [Mon May 6 16:55:08 CST 2024] <<<<<< START pulsar cluster in normal mode in row_format case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/row_format/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/row_format/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/row_format/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:55:09 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f34dec6-1790-47f1-883a-4bcf8df14f97 {"id":"3f34dec6-1790-47f1-883a-4bcf8df14f97","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985706} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1cde5eef 3f34dec6-1790-47f1-883a-4bcf8df14f97 /tidb/cdc/default/default/upstream/7365807457313954096 {"id":7365807457313954096,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f34dec6-1790-47f1-883a-4bcf8df14f97 {"id":"3f34dec6-1790-47f1-883a-4bcf8df14f97","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985706} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1cde5eef 3f34dec6-1790-47f1-883a-4bcf8df14f97 /tidb/cdc/default/default/upstream/7365807457313954096 {"id":7365807457313954096,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f34dec6-1790-47f1-883a-4bcf8df14f97 {"id":"3f34dec6-1790-47f1-883a-4bcf8df14f97","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985706} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1cde5eef 3f34dec6-1790-47f1-883a-4bcf8df14f97 /tidb/cdc/default/default/upstream/7365807457313954096 {"id":7365807457313954096,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:55:09 CST 2024] <<<<<< START pulsar cluster in sink_retry case >>>>>>' [Mon May 6 16:55:09 CST 2024] <<<<<< START pulsar cluster in sink_retry case >>>>>> + workdir=/tmp/tidb_cdc_test/sink_retry + cluster_type=normal + cd /tmp/tidb_cdc_test/sink_retry + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/sink_retry/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/sink_retry/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/sink_retry/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/sink_retry/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/sink_retry/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/sink_retry/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/sink_retry/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/sink_retry/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/sink_retry/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/sink_retry/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/sink_retry/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/sink_retry/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:55:09 CST 2024] <<<<<< START pulsar cluster in normal mode in sink_retry case >>>>>>' [Mon May 6 16:55:09 CST 2024] <<<<<< START pulsar cluster in normal mode in sink_retry case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/sink_retry/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/sink_retry/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/sink_retry/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:55:10 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e2c4443-c791-4f3a-b138-213e7e9ead57 {"id":"4e2c4443-c791-4f3a-b138-213e7e9ead57","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985708} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1cd6ece2 4e2c4443-c791-4f3a-b138-213e7e9ead57 /tidb/cdc/default/default/upstream/7365807448977155960 {"id":7365807448977155960,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e2c4443-c791-4f3a-b138-213e7e9ead57 {"id":"4e2c4443-c791-4f3a-b138-213e7e9ead57","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985708} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1cd6ece2 4e2c4443-c791-4f3a-b138-213e7e9ead57 /tidb/cdc/default/default/upstream/7365807448977155960 {"id":7365807448977155960,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e2c4443-c791-4f3a-b138-213e7e9ead57 {"id":"4e2c4443-c791-4f3a-b138-213e7e9ead57","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985708} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1cd6ece2 4e2c4443-c791-4f3a-b138-213e7e9ead57 /tidb/cdc/default/default/upstream/7365807448977155960 {"id":7365807448977155960,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + nc -z localhost 6651 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:55:11 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e836be77-54e2-4527-a58b-6463a4aeb342 {"id":"e836be77-54e2-4527-a58b-6463a4aeb342","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985708} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1ce402d3 e836be77-54e2-4527-a58b-6463a4aeb342 /tidb/cdc/default/default/upstream/7365807460734673785 {"id":7365807460734673785,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e836be77-54e2-4527-a58b-6463a4aeb342 {"id":"e836be77-54e2-4527-a58b-6463a4aeb342","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985708} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1ce402d3 e836be77-54e2-4527-a58b-6463a4aeb342 /tidb/cdc/default/default/upstream/7365807460734673785 {"id":7365807460734673785,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e836be77-54e2-4527-a58b-6463a4aeb342 {"id":"e836be77-54e2-4527-a58b-6463a4aeb342","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985708} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1ce402d3 e836be77-54e2-4527-a58b-6463a4aeb342 /tidb/cdc/default/default/upstream/7365807460734673785 {"id":7365807460734673785,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:55:11 CST 2024] <<<<<< START pulsar cluster in foreign_key case >>>>>>' [Mon May 6 16:55:11 CST 2024] <<<<<< START pulsar cluster in foreign_key case >>>>>> + workdir=/tmp/tidb_cdc_test/foreign_key + cluster_type=normal + cd /tmp/tidb_cdc_test/foreign_key + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/foreign_key/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/foreign_key/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/foreign_key/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/foreign_key/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/foreign_key/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/foreign_key/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/foreign_key/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/foreign_key/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/foreign_key/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/foreign_key/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/foreign_key/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/foreign_key/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:55:11 CST 2024] <<<<<< START pulsar cluster in normal mode in foreign_key case >>>>>>' [Mon May 6 16:55:11 CST 2024] <<<<<< START pulsar cluster in normal mode in foreign_key case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/foreign_key/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/foreign_key/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/foreign_key/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 [Mon May 6 16:55:11 CST 2024] <<<<<< START Pulsar consumer in kv_client_stream_reconnect case >>>>>> + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:55:12 CST 2024] <<<<<< run test case kafka_simple_handle_key_only success! >>>>>> + nc -z localhost 6650 + i=3 + '[' 3 -gt 20 ']' + sleep 2 public/default + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 ++ date + echo '[Mon May 6 16:55:13 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:55:13 CST 2024] <<<<<< pulsar is ready >>>>>> Create changefeed successfully! ID: 4a5f3ec8-57f1-4aed-8ba0-b04f1c6e7e79 Info: {"upstream_id":7365807435054545416,"namespace":"default","id":"4a5f3ec8-57f1-4aed-8ba0-b04f1c6e7e79","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T16:55:13.544431258+08:00","start_ts":449573212142960643,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573212142960643,"checkpoint_ts":449573212142960643,"checkpoint_time":"2024-05-06 16:55:03.060"} [Mon May 6 16:55:13 CST 2024] <<<<<< START Pulsar consumer in charset_gbk case >>>>>> + nc -z localhost 6650 + i=3 + '[' 3 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + i=4 + '[' 4 -gt 20 ']' + sleep 2 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:55:15 CST 2024] <<<<<< run test case kafka_simple_handle_key_only_avro success! >>>>>> + nc -z localhost 6650 + i=3 + '[' 3 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public public/default ++ date + echo '[Mon May 6 16:55:18 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:55:18 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.3368.out cli changefeed create --start-ts=449573212632645633 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-sink-retry-test-30553?protocol=canal-json&enable-tidb-extension=true' find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_simple_claim_check/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:55:18 CST 2024] <<<<<< run test case kafka_simple_claim_check success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 29.01 secs (126451898 bytes/sec) [Pipeline] { [Pipeline] cache Create changefeed successfully! ID: ca79647c-65a0-4a2d-8e88-c1b34a258bb9 Info: {"upstream_id":7365807457313954096,"namespace":"default","id":"ca79647c-65a0-4a2d-8e88-c1b34a258bb9","sink_uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-30553?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:55:19.147495239+08:00","start_ts":449573212632645633,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573212632645633,"checkpoint_ts":449573212632645633,"checkpoint_time":"2024-05-06 16:55:04.928"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [Mon May 6 16:55:20 CST 2024] <<<<<< START Pulsar consumer in sink_retry case >>>>>> Tenant not found Reason: Tenant not found + i=1 + '[' 1 -gt 20 ']' + sleep 2 public/default ++ date + echo '[Mon May 6 16:55:20 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:55:20 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.3168.out cli changefeed create --start-ts=449573212251226113 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-row-format-test-27260?protocol=canal-json&enable-tidb-extension=true' find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:55:21 CST 2024] <<<<<< run test case kafka_simple_claim_check_avro success! >>>>>> Create changefeed successfully! ID: 85e13829-85ff-4ea3-9ab4-1fdaad96d25a Info: {"upstream_id":7365807449580835332,"namespace":"default","id":"85e13829-85ff-4ea3-9ab4-1fdaad96d25a","sink_uri":"pulsar://127.0.0.1:6650/ticdc-row-format-test-27260?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:55:22.406046638+08:00","start_ts":449573212251226113,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573212251226113,"checkpoint_ts":449573212251226113,"checkpoint_time":"2024-05-06 16:55:03.473"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + /usr/local/pulsar/bin/pulsar-admin namespaces list public + set +x [Mon May 6 16:55:24 CST 2024] <<<<<< START Pulsar consumer in row_format case >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/canal_json_adapter_compatibility/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:55:24 CST 2024] <<<<<< run test case canal_json_adapter_compatibility success! >>>>>> public/default ++ date + echo '[Mon May 6 16:55:25 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:55:25 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.3445.out cli changefeed create --start-ts=449573213036609538 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-foreign-key-test-15629?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: ab410376-6548-4d14-9308-d20d312d9d6b Info: {"upstream_id":7365807460734673785,"namespace":"default","id":"ab410376-6548-4d14-9308-d20d312d9d6b","sink_uri":"pulsar://127.0.0.1:6650/ticdc-foreign-key-test-15629?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:55:26.17702406+08:00","start_ts":449573213036609538,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573213036609538,"checkpoint_ts":449573213036609538,"checkpoint_time":"2024-05-06 16:55:06.469"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [Mon May 6 16:55:27 CST 2024] <<<<<< START Pulsar consumer in foreign_key case >>>>>> table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/canal_json_basic/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table foreign_key.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 3-th check, retry later table foreign_key.finish_mark not exists for 2-th check, retry later table test.finish_mark exists check table exists success check diff successfully wait process cdc.test exit for 1-th time... table foreign_key.finish_mark not exists for 3-th check, retry later wait process cdc.test exit for 2-th time... table row_format.finish_mark not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 16:55:34 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 14.07 secs (260609704 bytes/sec) [Pipeline] { [Pipeline] cache table foreign_key.finish_mark not exists for 4-th check, retry later table row_format.finish_mark not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/canal_json_basic Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table foreign_key.finish_mark not exists for 5-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table row_format.finish_mark not exists for 3-th check, retry later table foreign_key.finish_mark not exists for 6-th check, retry later table row_format.finish_mark not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table foreign_key.finish_mark not exists for 7-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 5-th check, retry later table foreign_key.finish_mark not exists for 8-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 6-th check, retry later table foreign_key.finish_mark not exists for 9-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 7-th check, retry later table foreign_key.finish_mark not exists for 10-th check, retry later table row_format.finish_mark not exists for 8-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347709600018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0g2h-bv6sm, pid:2430, start at 2024-05-06 16:55:47.078721673 +0800 CST m=+6.398238000 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:57:47.085 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:55:47.083 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:45:47.083 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347709600018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0g2h-bv6sm, pid:2430, start at 2024-05-06 16:55:47.078721673 +0800 CST m=+6.398238000 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:57:47.085 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:55:47.083 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:45:47.083 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3476f8200014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0g2h-bv6sm, pid:2519, start at 2024-05-06 16:55:45.958621737 +0800 CST m=+5.213087497 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:57:45.966 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:55:45.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:45:45.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_basic/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_basic/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_basic/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 1-th time, retry later check diff successfully table foreign_key.finish_mark not exists for 11-th check, retry later wait process cdc.test exit for 1-th time... [Mon May 6 16:55:50 CST 2024] <<<<<< START cdc server in canal_json_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_basic.38593861.out server --log-file /tmp/tidb_cdc_test/canal_json_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_basic/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table row_format.finish_mark not exists for 9-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 16:55:51 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> table foreign_key.finish_mark exists check diff successfully start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... wait process cdc.test exit for 1-th time... table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "mysql.host"="127.0.0.1" "readallfields"="true" "mysql.port"="4000" "insertproportion"="0" "scanproportion"="0" "mysql.db"="sink_retry" "recordcount"="10" "workload"="core" "dotransactions"="false" "updateproportion"="0" "operationcount"="0" "readproportion"="0" "threadcount"="2" "requestdistribution"="uniform" "mysql.user"="root" ********************************************** Run finished, takes 4.224565ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3729.5, Avg(us): 771, Min(us): 522, Max(us): 1630, 95th(us): 2000, 99th(us): 2000 table row_format.finish_mark not exists for 10-th check, retry later wait process cdc.test exit for 2-th time... Verifying downstream PD is started... wait process cdc.test exit for 3-th time... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:55:54 CST 2024] <<<<<< run test case foreign_key success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:55:54 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ddc85cc3-d9b7-4228-8e0e-56b7e6a58cff {"id":"ddc85cc3-d9b7-4228-8e0e-56b7e6a58cff","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985751} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1d9981cc ddc85cc3-d9b7-4228-8e0e-56b7e6a58cff /tidb/cdc/default/default/upstream/7365807654589763581 {"id":7365807654589763581,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ddc85cc3-d9b7-4228-8e0e-56b7e6a58cff {"id":"ddc85cc3-d9b7-4228-8e0e-56b7e6a58cff","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985751} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1d9981cc ddc85cc3-d9b7-4228-8e0e-56b7e6a58cff /tidb/cdc/default/default/upstream/7365807654589763581 {"id":7365807654589763581,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ddc85cc3-d9b7-4228-8e0e-56b7e6a58cff {"id":"ddc85cc3-d9b7-4228-8e0e-56b7e6a58cff","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985751} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1d9981cc ddc85cc3-d9b7-4228-8e0e-56b7e6a58cff /tidb/cdc/default/default/upstream/7365807654589763581 {"id":7365807654589763581,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:55:54 CST 2024] <<<<<< START pulsar cluster in canal_json_basic case >>>>>>' [Mon May 6 16:55:54 CST 2024] <<<<<< START pulsar cluster in canal_json_basic case >>>>>> + workdir=/tmp/tidb_cdc_test/canal_json_basic + cluster_type=normal + cd /tmp/tidb_cdc_test/canal_json_basic + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/canal_json_basic/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/canal_json_basic/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/canal_json_basic/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/canal_json_basic/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/canal_json_basic/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/canal_json_basic/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/canal_json_basic/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/canal_json_basic/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/canal_json_basic/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/canal_json_basic/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/canal_json_basic/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/canal_json_basic/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:55:54 CST 2024] <<<<<< START pulsar cluster in normal mode in canal_json_basic case >>>>>>' [Mon May 6 16:55:54 CST 2024] <<<<<< START pulsar cluster in normal mode in canal_json_basic case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/canal_json_basic/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/canal_json_basic/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/canal_json_basic/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 table row_format.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 17.36 secs (211268352 bytes/sec) [Pipeline] { wait process cdc.test exit for 2-th time... + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 [Pipeline] cache wait process cdc.test exit for 3-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:55:57 CST 2024] <<<<<< run test case row_format success! >>>>>> + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: pulsar... <<================= ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) public/default ++ date + echo '[Mon May 6 16:56:03 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:56:03 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_basic.cli.4339.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-canal-json-basic?protocol=canal-json&enable-tidb-extension=true' =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3477ebac0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-h5k3-ghhrj, pid:4652, start at 2024-05-06 16:56:01.545048813 +0800 CST m=+5.284208691 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:01.552 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:01.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:01.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3477ebac0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-h5k3-ghhrj, pid:4652, start at 2024-05-06 16:56:01.545048813 +0800 CST m=+5.284208691 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:01.552 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:01.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:01.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3477ec940013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-h5k3-ghhrj, pid:4728, start at 2024-05-06 16:56:01.604869919 +0800 CST m=+5.293554889 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:01.611 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:01.573 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:01.573 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Create changefeed successfully! ID: e5927663-71d2-4af0-acc0-009051067859 Info: {"upstream_id":7365807654589763581,"namespace":"default","id":"e5927663-71d2-4af0-acc0-009051067859","sink_uri":"pulsar://127.0.0.1:6650/ticdc-canal-json-basic?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:56:04.709625379+08:00","start_ts":449573228126404610,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573228126404610,"checkpoint_ts":449573228126404610,"checkpoint_time":"2024-05-06 16:56:04.032"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + set +x [Mon May 6 16:56:06 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.60686070.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:56:09 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/46873cf3-c549-422b-979e-b09413db3f37 {"id":"46873cf3-c549-422b-979e-b09413db3f37","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985766} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1dd245ce 46873cf3-c549-422b-979e-b09413db3f37 /tidb/cdc/default/default/upstream/7365807719425627677 {"id":7365807719425627677,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/46873cf3-c549-422b-979e-b09413db3f37 {"id":"46873cf3-c549-422b-979e-b09413db3f37","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985766} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1dd245ce 46873cf3-c549-422b-979e-b09413db3f37 /tidb/cdc/default/default/upstream/7365807719425627677 {"id":7365807719425627677,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/46873cf3-c549-422b-979e-b09413db3f37 {"id":"46873cf3-c549-422b-979e-b09413db3f37","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985766} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1dd245ce 46873cf3-c549-422b-979e-b09413db3f37 /tidb/cdc/default/default/upstream/7365807719425627677 {"id":7365807719425627677,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:56:09 CST 2024] <<<<<< START pulsar cluster in ddl_manager case >>>>>>' [Mon May 6 16:56:09 CST 2024] <<<<<< START pulsar cluster in ddl_manager case >>>>>> + workdir=/tmp/tidb_cdc_test/ddl_manager + cluster_type=normal + cd /tmp/tidb_cdc_test/ddl_manager + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_manager/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/ddl_manager/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/ddl_manager/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_manager/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/ddl_manager/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/ddl_manager/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/ddl_manager/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/ddl_manager/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_manager/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/ddl_manager/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/ddl_manager/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/ddl_manager/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:56:09 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_manager case >>>>>>' [Mon May 6 16:56:09 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_manager case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/ddl_manager/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/ddl_manager/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/ddl_manager/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 [Mon May 6 16:56:11 CST 2024] <<<<<< START Pulsar consumer in canal_json_basic case >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) public/default ++ date + echo '[Mon May 6 16:56:17 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:56:17 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.6537.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-16502?protocol=canal-json&enable-tidb-extension=true' -c=ddl-manager ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7365807719425627677,"namespace":"default","id":"ddl-manager","sink_uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-16502?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:56:18.254920709+08:00","start_ts":449573231673999362,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573231673999362,"checkpoint_ts":449573231673999362,"checkpoint_time":"2024-05-06 16:56:17.565"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + set +x [Mon May 6 16:56:19 CST 2024] <<<<<< START Pulsar consumer in ddl_manager case >>>>>> Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3478fdfc000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-g720-bjdgs, pid:4710, start at 2024-05-06 16:56:19.089232506 +0800 CST m=+5.325282382 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:19.098 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:19.071 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:19.071 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34790c980016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-4q3d-6b3lc, pid:4986, start at 2024-05-06 16:56:20.027074689 +0800 CST m=+5.280298566 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:20.036 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:20.006 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:20.006 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3478fdfc000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-g720-bjdgs, pid:4710, start at 2024-05-06 16:56:19.089232506 +0800 CST m=+5.325282382 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:19.098 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:19.071 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:19.071 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3478ff8c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-g720-bjdgs, pid:4795, start at 2024-05-06 16:56:19.183666999 +0800 CST m=+5.366602493 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:19.191 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:19.171 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:19.171 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.finish_mark not exists for 4-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34790c980016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-4q3d-6b3lc, pid:4986, start at 2024-05-06 16:56:20.027074689 +0800 CST m=+5.280298566 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:20.036 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:20.006 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:20.006 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34790d400016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-4q3d-6b3lc, pid:5069, start at 2024-05-06 16:56:20.093931935 +0800 CST m=+5.297735792 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:20.100 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:20.098 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:20.098 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/split_region/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.6190.out cli tso query --pd=http://127.0.0.1:2379 table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 16:56:22 CST 2024] <<<<<< run test case sink_retry success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 5-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.6448.out cli tso query --pd=http://127.0.0.1:2379 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 24.60 secs (149105057 bytes/sec) [Pipeline] { [Pipeline] cache table test.finish_mark not exists for 6-th check, retry later + set +x + tso='449573233729470465 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573233729470465 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + set +x + tso='449573233471258626 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573233471258626 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:56:27 CST 2024] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.64906492.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 16:56:27 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.62406242.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 7-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:56:30 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a5dd009-e835-40f0-844e-f1836d128daf {"id":"9a5dd009-e835-40f0-844e-f1836d128daf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985787} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e16cbfa 9a5dd009-e835-40f0-844e-f1836d128daf /tidb/cdc/default/default/upstream/7365807799984319224 {"id":7365807799984319224,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a5dd009-e835-40f0-844e-f1836d128daf {"id":"9a5dd009-e835-40f0-844e-f1836d128daf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985787} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e16cbfa 9a5dd009-e835-40f0-844e-f1836d128daf /tidb/cdc/default/default/upstream/7365807799984319224 {"id":7365807799984319224,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a5dd009-e835-40f0-844e-f1836d128daf {"id":"9a5dd009-e835-40f0-844e-f1836d128daf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985787} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e16cbfa 9a5dd009-e835-40f0-844e-f1836d128daf /tidb/cdc/default/default/upstream/7365807799984319224 {"id":7365807799984319224,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:56:30 CST 2024] <<<<<< START pulsar cluster in split_region case >>>>>>' [Mon May 6 16:56:30 CST 2024] <<<<<< START pulsar cluster in split_region case >>>>>> + workdir=/tmp/tidb_cdc_test/split_region + cluster_type=oauth + cd /tmp/tidb_cdc_test/split_region + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/split_region/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/split_region/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/split_region/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/split_region/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/split_region/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/split_region/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/split_region/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/split_region/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/split_region/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/split_region/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/split_region/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/split_region/pulsar_standalone.conf + pulsar_port=6650 + '[' oauth == mtls ']' + '[' oauth == oauth ']' + echo 'Waiting for oauth2 server to be ready...' Waiting for oauth2 server to be ready... + i=0 + nc -z localhost 9096 + oauth2-server + gen_oauth_config + echo ' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/split_region/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/split_region/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ date + echo '[Mon May 6 16:56:30 CST 2024] <<<<<< START pulsar cluster in oauth mode in split_region case >>>>>>' [Mon May 6 16:56:30 CST 2024] <<<<<< START pulsar cluster in oauth mode in split_region case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/split_region/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/split_region/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/split_region/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 table test.finish_mark not exists for 8-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:56:30 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09edc5cf-3a08-4251-ab1a-4c2cdc6f6ad4 {"id":"09edc5cf-3a08-4251-ab1a-4c2cdc6f6ad4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985787} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e12f0f3 09edc5cf-3a08-4251-ab1a-4c2cdc6f6ad4 /tidb/cdc/default/default/upstream/7365807799342296700 {"id":7365807799342296700,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09edc5cf-3a08-4251-ab1a-4c2cdc6f6ad4 {"id":"09edc5cf-3a08-4251-ab1a-4c2cdc6f6ad4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985787} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e12f0f3 09edc5cf-3a08-4251-ab1a-4c2cdc6f6ad4 /tidb/cdc/default/default/upstream/7365807799342296700 {"id":7365807799342296700,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09edc5cf-3a08-4251-ab1a-4c2cdc6f6ad4 {"id":"09edc5cf-3a08-4251-ab1a-4c2cdc6f6ad4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985787} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e12f0f3 09edc5cf-3a08-4251-ab1a-4c2cdc6f6ad4 /tidb/cdc/default/default/upstream/7365807799342296700 {"id":7365807799342296700,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:56:30 CST 2024] <<<<<< START pulsar cluster in ddl_puller_lag case >>>>>>' [Mon May 6 16:56:30 CST 2024] <<<<<< START pulsar cluster in ddl_puller_lag case >>>>>> + workdir=/tmp/tidb_cdc_test/ddl_puller_lag + cluster_type=normal + cd /tmp/tidb_cdc_test/ddl_puller_lag + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/ddl_puller_lag/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/ddl_puller_lag/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/ddl_puller_lag/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/ddl_puller_lag/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/ddl_puller_lag/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/ddl_puller_lag/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/ddl_puller_lag/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:56:30 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_puller_lag case >>>>>>' [Mon May 6 16:56:30 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_puller_lag case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/ddl_puller_lag/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/ddl_puller_lag/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/ddl_puller_lag/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 9-th check, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3479d1d8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq, pid:4423, start at 2024-05-06 16:56:32.660985604 +0800 CST m=+7.921540284 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:32.674 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:32.681 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:32.681 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3479d1d8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq, pid:4423, start at 2024-05-06 16:56:32.660985604 +0800 CST m=+7.921540284 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:32.674 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:32.681 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:32.681 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3479c42c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq, pid:4456, start at 2024-05-06 16:56:31.801387894 +0800 CST m=+6.970234555 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:31.811 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:31.804 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:31.804 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 10-th check, retry later table test.finish_mark not exists for 11-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... public/default ++ date + echo '[Mon May 6 16:56:36 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:56:36 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.6976.out cli changefeed create --start-ts=449573233729470465 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-split-region-test-29733?protocol=canal-json&enable-tidb-extension=true' --config /tmp/tidb_cdc_test/split_region/pulsar_test.toml public/default ++ date + echo '[Mon May 6 16:56:36 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:56:36 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.6701.out cli changefeed create --start-ts=449573233471258626 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-ddl-puller-lag-test-366?protocol=canal-json&enable-tidb-extension=true' + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5759.out cli tso query --pd=http://127.0.0.1:2379 Create changefeed successfully! ID: d5485783-99b4-4f8c-bfe3-050389615784 Info: {"upstream_id":7365807799342296700,"namespace":"default","id":"d5485783-99b4-4f8c-bfe3-050389615784","sink_uri":"pulsar://127.0.0.1:6650/ticdc-ddl-puller-lag-test-366?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:56:37.784213696+08:00","start_ts":449573233471258626,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573233471258626,"checkpoint_ts":449573233471258626,"checkpoint_time":"2024-05-06 16:56:24.421"} PASS Create changefeed successfully! ID: f6ed6911-fc3d-4c59-a8ab-91243adc3841 Info: {"upstream_id":7365807799984319224,"namespace":"default","id":"f6ed6911-fc3d-4c59-a8ab-91243adc3841","sink_uri":"pulsar://127.0.0.1:6650/ticdc-split-region-test-29733?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:56:37.5888032+08:00","start_ts":449573233729470465,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30,"oauth2":{"oauth2-issuer-url":"http://localhost:9096","oauth2-audience":"cdc-api-uri","oauth2-private-key":"/tmp/tidb_cdc_test/split_region/credential.json","oauth2-client-id":"1234"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573233729470465,"checkpoint_ts":449573233729470465,"checkpoint_time":"2024-05-06 16:56:25.406"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... wait process 6073 exit for 1-th time... wait process 6073 exit for 2-th time... wait process 6073 exit for 3-th time... wait process 6073 exit for 4-th time... wait process 6073 exit for 5-th time... wait process 6073 exit for 6-th time... coverage: 2.4% of statements in github.com/pingcap/tiflow/... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (6073) - No such process wait process 6073 exit for 7-th time... process 6073 already exit [Mon May 6 16:56:38 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.66766678.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449573236790001667 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573236790001667 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:56:38 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.57955797.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x [Mon May 6 16:56:39 CST 2024] <<<<<< START Pulsar consumer in split_region case >>>>>> table split_region.test1 not exists for 1-th check, retry later table test.finish_mark not exists for 12-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + set +x [Mon May 6 16:56:39 CST 2024] <<<<<< START Pulsar consumer in ddl_puller_lag case >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table split_region.test1 exists table split_region.test2 not exists for 1-th check, retry later table test.finish_mark not exists for 13-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:56:42 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0053dbe-a6cd-4644-8fa4-b56043cb9f2e {"id":"e0053dbe-a6cd-4644-8fa4-b56043cb9f2e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985799} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e3d5bdc e0053dbe-a6cd-4644-8fa4-b56043cb9f2e /tidb/cdc/default/default/upstream/7365807836542180736 {"id":7365807836542180736,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0053dbe-a6cd-4644-8fa4-b56043cb9f2e {"id":"e0053dbe-a6cd-4644-8fa4-b56043cb9f2e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985799} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e3d5bdc e0053dbe-a6cd-4644-8fa4-b56043cb9f2e /tidb/cdc/default/default/upstream/7365807836542180736 {"id":7365807836542180736,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0053dbe-a6cd-4644-8fa4-b56043cb9f2e {"id":"e0053dbe-a6cd-4644-8fa4-b56043cb9f2e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985799} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e3d5bdc e0053dbe-a6cd-4644-8fa4-b56043cb9f2e /tidb/cdc/default/default/upstream/7365807836542180736 {"id":7365807836542180736,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:56:42 CST 2024] <<<<<< START pulsar cluster in drop_many_tables case >>>>>>' [Mon May 6 16:56:42 CST 2024] <<<<<< START pulsar cluster in drop_many_tables case >>>>>> + workdir=/tmp/tidb_cdc_test/drop_many_tables + cluster_type=normal + cd /tmp/tidb_cdc_test/drop_many_tables + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/drop_many_tables/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/drop_many_tables/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/drop_many_tables/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/drop_many_tables/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/drop_many_tables/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/drop_many_tables/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/drop_many_tables/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/drop_many_tables/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/drop_many_tables/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/drop_many_tables/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/drop_many_tables/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/drop_many_tables/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:56:42 CST 2024] <<<<<< START pulsar cluster in normal mode in drop_many_tables case >>>>>>' [Mon May 6 16:56:42 CST 2024] <<<<<< START pulsar cluster in normal mode in drop_many_tables case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/drop_many_tables/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/drop_many_tables/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/drop_many_tables/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:56:42 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact table test.finish_mark not exists for 14-th check, retry later + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365807719425627677 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-16502?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 16:56:18.254920709 +0800 CST StartTs:449573231673999362 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d081b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573231844392965} {CheckpointTs:449573232512860175 MinTableBarrierTs:449573232512860175 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/14fd3137-754f-4b85-8b88-271cedd0f20c {"id":"14fd3137-754f-4b85-8b88-271cedd0f20c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985798} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1dd248f1 14fd3137-754f-4b85-8b88-271cedd0f20c /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365807719425627677,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-16502?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T16:56:18.254920709+08:00","start-ts":449573231673999362,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573231844392965} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449573232512860175,"min-table-barrier-ts":449573232512860175,"admin-job-type":0} /tidb/cdc/default/default/task/position/14fd3137-754f-4b85-8b88-271cedd0f20c/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365807719425627677 {"id":7365807719425627677,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365807719425627677 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-16502?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 16:56:18.254920709 +0800 CST StartTs:449573231673999362 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d081b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573231844392965} {CheckpointTs:449573232512860175 MinTableBarrierTs:449573232512860175 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/14fd3137-754f-4b85-8b88-271cedd0f20c {"id":"14fd3137-754f-4b85-8b88-271cedd0f20c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985798} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1dd248f1 14fd3137-754f-4b85-8b88-271cedd0f20c /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365807719425627677,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-16502?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T16:56:18.254920709+08:00","start-ts":449573231673999362,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573231844392965} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449573232512860175,"min-table-barrier-ts":449573232512860175,"admin-job-type":0} /tidb/cdc/default/default/task/position/14fd3137-754f-4b85-8b88-271cedd0f20c/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365807719425627677 {"id":7365807719425627677,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365807719425627677 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-16502?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 16:56:18.254920709 +0800 CST StartTs:449573231673999362 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d081b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573231844392965} {CheckpointTs:449573232512860175 MinTableBarrierTs:449573232512860175 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/14fd3137-754f-4b85-8b88-271cedd0f20c {"id":"14fd3137-754f-4b85-8b88-271cedd0f20c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985798} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1dd248f1 14fd3137-754f-4b85-8b88-271cedd0f20c /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365807719425627677,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-16502?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T16:56:18.254920709+08:00","start-ts":449573231673999362,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573231844392965} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449573232512860175,"min-table-barrier-ts":449573232512860175,"admin-job-type":0} /tidb/cdc/default/default/task/position/14fd3137-754f-4b85-8b88-271cedd0f20c/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365807719425627677 + grep -q 'etcd info' {"id":7365807719425627677,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Mon May 6 16:56:42 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.67466748.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table split_region.test2 exists check diff failed 1-th time, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table test.finish_mark not exists for 15-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully + nc -z localhost 6650 + i=3 + '[' 3 -gt 20 ']' + sleep 2 table test.finish_mark not exists for 16-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:56:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365807719425627677 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-16502?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 16:56:18.254920709 +0800 CST StartTs:449573231673999362 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d081b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573231844392965} {CheckpointTs:449573232512860175 MinTableBarrierTs:449573232512860175 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/14fd3137-754f-4b85-8b88-271cedd0f20c {"id":"14fd3137-754f-4b85-8b88-271cedd0f20c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985798} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1dd248f1 14fd3137-754f-4b85-8b88-271cedd0f20c /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365807719425627677,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-16502?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T16:56:18.254920709+08:00","start-ts":449573231673999362,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573231844392965} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449573232643932185,"min-table-barrier-ts":449573232643932185,"admin-job-type":0} /tidb/cdc/default/default/task/position/14fd3137-754f-4b85-8b88-271cedd0f20c/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365807719425627677 {"id":7365807719425627677,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365807719425627677 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-16502?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 16:56:18.254920709 +0800 CST StartTs:449573231673999362 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d081b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573231844392965} {CheckpointTs:449573232512860175 MinTableBarrierTs:449573232512860175 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/14fd3137-754f-4b85-8b88-271cedd0f20c {"id":"14fd3137-754f-4b85-8b88-271cedd0f20c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985798} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1dd248f1 14fd3137-754f-4b85-8b88-271cedd0f20c /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365807719425627677,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-16502?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T16:56:18.254920709+08:00","start-ts":449573231673999362,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573231844392965} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449573232643932185,"min-table-barrier-ts":449573232643932185,"admin-job-type":0} /tidb/cdc/default/default/task/position/14fd3137-754f-4b85-8b88-271cedd0f20c/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365807719425627677 {"id":7365807719425627677,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365807719425627677 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-16502?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 16:56:18.254920709 +0800 CST StartTs:449573231673999362 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d081b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573231844392965} {CheckpointTs:449573232512860175 MinTableBarrierTs:449573232512860175 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/14fd3137-754f-4b85-8b88-271cedd0f20c {"id":"14fd3137-754f-4b85-8b88-271cedd0f20c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985798} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1dd248f1 14fd3137-754f-4b85-8b88-271cedd0f20c /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365807719425627677,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-16502?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T16:56:18.254920709+08:00","start-ts":449573231673999362,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573231844392965} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449573232643932185,"min-table-barrier-ts":449573232643932185,"admin-job-type":0} /tidb/cdc/default/default/task/position/14fd3137-754f-4b85-8b88-271cedd0f20c/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365807719425627677 + grep -q 'etcd info' {"id":7365807719425627677,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later + nc -z localhost 6650 + i=4 + '[' 4 -gt 20 ']' + sleep 2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347abc740013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8, pid:4702, start at 2024-05-06 16:56:47.670989835 +0800 CST m=+5.288060694 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:47.680 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:47.645 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:47.645 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 18.93 secs (193752715 bytes/sec) [Pipeline] { [Pipeline] cache table test.finish_mark not exists for 17-th check, retry later table ddl_manager.finish_mark not exists for 2-th check, retry later + nc -z localhost 6650 + i=5 + '[' 5 -gt 20 ']' + sleep 2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347abc740013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8, pid:4702, start at 2024-05-06 16:56:47.670989835 +0800 CST m=+5.288060694 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:47.680 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:47.645 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:47.645 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347abea40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8, pid:4791, start at 2024-05-06 16:56:47.828651777 +0800 CST m=+5.394675659 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:47.835 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:47.835 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:47.835 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark exists check diff successfully table ddl_manager.finish_mark not exists for 3-th check, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6189.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 4-th check, retry later + set +x + tso='449573240961761281 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573240961761281 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "dotransactions"="false" "recordcount"="20" "readallfields"="true" "mysql.user"="root" "operationcount"="0" "workload"="core" "updateproportion"="0" "requestdistribution"="uniform" "mysql.port"="4000" "scanproportion"="0" "threadcount"="4" "mysql.host"="127.0.0.1" "mysql.db"="changefeed_error" "insertproportion"="0" "readproportion"="0" ********************************************** Run finished, takes 11.082582ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3148.1, Avg(us): 2055, Min(us): 1103, Max(us): 5033, 95th(us): 6000, 99th(us): 6000 [Mon May 6 16:56:54 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.62576259.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 5-th check, retry later table test.finish_mark not exists for 1-th check, retry later Tenant not found Reason: Tenant not found + i=1 + '[' 1 -gt 20 ']' + sleep 2 check diff failed 1-th time, retry later table test.finish_mark not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 6-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:56:58 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/510f285f-5949-4421-83fd-bb2de9f35d0d {"id":"510f285f-5949-4421-83fd-bb2de9f35d0d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985815} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e8c84ed 510f285f-5949-4421-83fd-bb2de9f35d0d /tidb/cdc/default/default/upstream/7365807920150637154 {"id":7365807920150637154,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/510f285f-5949-4421-83fd-bb2de9f35d0d {"id":"510f285f-5949-4421-83fd-bb2de9f35d0d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985815} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e8c84ed 510f285f-5949-4421-83fd-bb2de9f35d0d /tidb/cdc/default/default/upstream/7365807920150637154 {"id":7365807920150637154,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/510f285f-5949-4421-83fd-bb2de9f35d0d {"id":"510f285f-5949-4421-83fd-bb2de9f35d0d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985815} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e8c84ed 510f285f-5949-4421-83fd-bb2de9f35d0d /tidb/cdc/default/default/upstream/7365807920150637154 {"id":7365807920150637154,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:56:58 CST 2024] <<<<<< START pulsar cluster in changefeed_error case >>>>>>' [Mon May 6 16:56:58 CST 2024] <<<<<< START pulsar cluster in changefeed_error case >>>>>> + workdir=/tmp/tidb_cdc_test/changefeed_error + cluster_type=normal + cd /tmp/tidb_cdc_test/changefeed_error + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_error/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/changefeed_error/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/changefeed_error/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_error/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/changefeed_error/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/changefeed_error/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/changefeed_error/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/changefeed_error/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_error/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/changefeed_error/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/changefeed_error/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/changefeed_error/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:56:58 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_error case >>>>>>' [Mon May 6 16:56:58 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_error case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/changefeed_error/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/changefeed_error/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/changefeed_error/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + /usr/local/pulsar/bin/pulsar-admin namespaces list public check diff failed 2-th time, retry later table test.finish_mark not exists for 3-th check, retry later table ddl_manager.finish_mark not exists for 7-th check, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table test.finish_mark not exists for 4-th check, retry later check diff successfully table ddl_manager.finish_mark not exists for 8-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 16:57:02 CST 2024] <<<<<< run test case split_region success! >>>>>> table test.finish_mark not exists for 5-th check, retry later table ddl_manager.finish_mark not exists for 9-th check, retry later Tenant not found Reason: Tenant not found + i=1 + '[' 1 -gt 20 ']' + sleep 2 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 13.21 secs (277562027 bytes/sec) [Pipeline] { [Pipeline] cache table test.finish_mark not exists for 6-th check, retry later table ddl_manager.finish_mark not exists for 10-th check, retry later public/default ++ date + echo '[Mon May 6 16:57:06 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:57:06 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.6347.out cli changefeed create --start-ts=449573236790001667 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-drop-tables-test-29723?protocol=canal-json&enable-tidb-extension=true' + /usr/local/pulsar/bin/pulsar-admin namespaces list public table test.finish_mark not exists for 7-th check, retry later table ddl_manager.finish_mark not exists for 11-th check, retry later Create changefeed successfully! ID: c9b23dff-d607-46bf-9ca3-fa50ee8104cb Info: {"upstream_id":7365807836542180736,"namespace":"default","id":"c9b23dff-d607-46bf-9ca3-fa50ee8104cb","sink_uri":"pulsar://127.0.0.1:6650/ticdc-drop-tables-test-29723?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:57:07.960816514+08:00","start_ts":449573236790001667,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573236790001667,"checkpoint_ts":449573236790001667,"checkpoint_time":"2024-05-06 16:56:37.081"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... public/default ++ date + echo '[Mon May 6 16:57:08 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:57:08 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6813.out cli changefeed create --start-ts=449573240961761281 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-sink-retry-test-10091?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error table test.finish_mark not exists for 8-th check, retry later table ddl_manager.finish_mark not exists for 12-th check, retry later Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7365807920150637154,"namespace":"default","id":"changefeed-error","sink_uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-10091?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:57:09.737816834+08:00","start_ts":449573240961761281,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573240961761281,"checkpoint_ts":449573240961761281,"checkpoint_time":"2024-05-06 16:56:52.995"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [Mon May 6 16:57:09 CST 2024] <<<<<< START Pulsar consumer in drop_many_tables case >>>>>> table test.finish_mark exists check diff successfully table ddl_manager.finish_mark not exists for 13-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... \033[0;36m<<< Run all test success >>>\033[0m + set +x [Mon May 6 16:57:12 CST 2024] <<<<<< START Pulsar consumer in changefeed_error case >>>>>> check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449573240961761281, "checkpoint_time": "2024-05-06 16:56:52.995", "error": { "time": "2024-05-06T16:57:09.819394807+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449573240961760981 is earlier than or equal to GC safepoint at 449573240961761281" } }' + echo '{ "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449573240961761281, "checkpoint_time": "2024-05-06 16:56:52.995", "error": { "time": "2024-05-06T16:57:09.819394807+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449573240961760981 is earlier than or equal to GC safepoint at 449573240961761281" } }' { "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449573240961761281, "checkpoint_time": "2024-05-06 16:56:52.995", "error": { "time": "2024-05-06T16:57:09.819394807+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449573240961760981 is earlier than or equal to GC safepoint at 449573240961761281" } } ++ echo '{' '"upstream_id":' 7365807920150637154, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449573240961761281, '"checkpoint_time":' '"2024-05-06' '16:56:52.995",' '"error":' '{' '"time":' '"2024-05-06T16:57:09.819394807+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449573240961760981 is earlier than or equal to GC safepoint at '449573240961761281"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365807920150637154, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449573240961761281, '"checkpoint_time":' '"2024-05-06' '16:56:52.995",' '"error":' '{' '"time":' '"2024-05-06T16:57:09.819394807+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449573240961760981 is earlier than or equal to GC safepoint at '449573240961761281"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449573240961760981 is earlier than or equal to GC safepoint at 449573240961761281' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449573240961760981 is earlier than or equal to GC safepoint at 449573240961761281 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6950.out cli changefeed resume -c changefeed-error PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 14-th check, retry later table drop_tables.c not exists for 1-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 7.39 secs (496112676 bytes/sec) [Pipeline] { [Pipeline] cache + set +x table changefeed_error.usertable not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:57:15 CST 2024] <<<<<< run test case canal_json_basic success! >>>>>> table ddl_manager.finish_mark not exists for 15-th check, retry later table drop_tables.c not exists for 2-th check, retry later table changefeed_error.usertable exists check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 16-th check, retry later table drop_tables.c not exists for 3-th check, retry later table ddl_manager.finish_mark not exists for 17-th check, retry later check diff successfully ***************** properties ***************** "dotransactions"="false" "threadcount"="4" "scanproportion"="0" "mysql.db"="changefeed_error" "operationcount"="0" "workload"="core" "recordcount"="20" "requestdistribution"="uniform" "readallfields"="true" "insertproportion"="0" "mysql.user"="root" "updateproportion"="0" "readproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" ********************************************** Run finished, takes 5.03544ms INSERT - Takes(s): 0.0, Count: 20, OPS: 5721.8, Avg(us): 863, Min(us): 450, Max(us): 2451, 95th(us): 3000, 99th(us): 3000 check diff successfully {"id":"510f285f-5949-4421-83fd-bb2de9f35d0d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985815} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/510f285f-5949-4421-83fd-bb2de9f35d0d + [[ /tidb/cdc/default/__cdc_meta__/capture/510f285f-5949-4421-83fd-bb2de9f35d0d =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/510f285f-5949-4421-83fd-bb2de9f35d0d' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/510f285f-5949-4421-83fd-bb2de9f35d0d + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later table drop_tables.c exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 18-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/510f285f-5949-4421-83fd-bb2de9f35d0d + [[ /tidb/cdc/default/__cdc_meta__/capture/510f285f-5949-4421-83fd-bb2de9f35d0d =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/510f285f-5949-4421-83fd-bb2de9f35d0d' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/510f285f-5949-4421-83fd-bb2de9f35d0d + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:57:22 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> table ddl_manager.finish_mark not exists for 19-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/canal_json_content_compatible/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 20-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/510f285f-5949-4421-83fd-bb2de9f35d0d + [[ /tidb/cdc/default/__cdc_meta__/capture/510f285f-5949-4421-83fd-bb2de9f35d0d =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/510f285f-5949-4421-83fd-bb2de9f35d0d' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/510f285f-5949-4421-83fd-bb2de9f35d0d + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later table ddl_manager.finish_mark not exists for 21-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/canal_json_content_compatible Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 22-th check, retry later table ddl_manager.finish_mark not exists for 23-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 16:57:31 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.72177219.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 24-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:57:34 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92b53500-7774-49ba-a04f-589b2a0f1645 {"id":"92b53500-7774-49ba-a04f-589b2a0f1645","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985852} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e8c862c 92b53500-7774-49ba-a04f-589b2a0f1645 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365807920150637154,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-10091?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T16:57:09.737816834+08:00","start-ts":449573240961761281,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-06T16:57:32.364410604+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573251277127683} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449573247371182088,"min-table-barrier-ts":449573247895207982,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365807920150637154 {"id":7365807920150637154,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92b53500-7774-49ba-a04f-589b2a0f1645 {"id":"92b53500-7774-49ba-a04f-589b2a0f1645","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985852} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e8c862c 92b53500-7774-49ba-a04f-589b2a0f1645 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365807920150637154,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-10091?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T16:57:09.737816834+08:00","start-ts":449573240961761281,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-06T16:57:32.364410604+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573251277127683} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449573247371182088,"min-table-barrier-ts":449573247895207982,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365807920150637154 {"id":7365807920150637154,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92b53500-7774-49ba-a04f-589b2a0f1645 {"id":"92b53500-7774-49ba-a04f-589b2a0f1645","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985852} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e8c862c 92b53500-7774-49ba-a04f-589b2a0f1645 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365807920150637154,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-10091?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T16:57:09.737816834+08:00","start-ts":449573240961761281,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-06T16:57:32.364410604+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573251277127683} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449573247371182088,"min-table-barrier-ts":449573247895207982,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365807920150637154 {"id":7365807920150637154,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449573247371182088, "checkpoint_time": "2024-05-06 16:57:17.445", "error": { "time": "2024-05-06T16:57:32.364410604+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449573247371182088, "checkpoint_time": "2024-05-06 16:57:17.445", "error": { "time": "2024-05-06T16:57:32.364410604+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449573247371182088, "checkpoint_time": "2024-05-06 16:57:17.445", "error": { "time": "2024-05-06T16:57:32.364410604+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7365807920150637154, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449573247371182088, '"checkpoint_time":' '"2024-05-06' '16:57:17.445",' '"error":' '{' '"time":' '"2024-05-06T16:57:32.364410604+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7365807920150637154, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449573247371182088, '"checkpoint_time":' '"2024-05-06' '16:57:17.445",' '"error":' '{' '"time":' '"2024-05-06T16:57:32.364410604+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7333.out cli changefeed remove -c changefeed-error ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Changefeed remove successfully. ID: changefeed-error CheckpointTs: 449573247371182088 SinkURI: pulsar://127.0.0.1:6650/ticdc-sink-retry-test-10091?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 25-th check, retry later + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 26-th check, retry later wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 16:57:38 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.74347436.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 27-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347dd8e4001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0g2h-bv6sm, pid:5798, start at 2024-05-06 16:57:38.664824899 +0800 CST m=+5.206109675 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:38.674 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:38.666 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:38.666 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347dd8e4001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0g2h-bv6sm, pid:5798, start at 2024-05-06 16:57:38.664824899 +0800 CST m=+5.206109675 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:38.674 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:38.666 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:38.666 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347dd9840013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0g2h-bv6sm, pid:5888, start at 2024-05-06 16:57:38.681106457 +0800 CST m=+5.166136836 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:38.689 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:38.657 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:38.657 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_content_compatible/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_content_compatible/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_content_compatible/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_content_compatible/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_content_compatible/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 28-th check, retry later [Mon May 6 16:57:41 CST 2024] <<<<<< START cdc server in canal_json_content_compatible case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_content_compatible.72087210.out server --log-file /tmp/tidb_cdc_test/canal_json_content_compatible/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_content_compatible/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:57:42 GMT < Content-Length: 884 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73ba5aa8-8a4d-42bd-9a81-136192ad8379 {"id":"73ba5aa8-8a4d-42bd-9a81-136192ad8379","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985859} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e8c867d 73ba5aa8-8a4d-42bd-9a81-136192ad8379 /tidb/cdc/default/default/upstream/7365807920150637154 {"id":7365807920150637154,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73ba5aa8-8a4d-42bd-9a81-136192ad8379 {"id":"73ba5aa8-8a4d-42bd-9a81-136192ad8379","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985859} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e8c867d 73ba5aa8-8a4d-42bd-9a81-136192ad8379 /tidb/cdc/default/default/upstream/7365807920150637154 {"id":7365807920150637154,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73ba5aa8-8a4d-42bd-9a81-136192ad8379 {"id":"73ba5aa8-8a4d-42bd-9a81-136192ad8379","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985859} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e8c867d 73ba5aa8-8a4d-42bd-9a81-136192ad8379 /tidb/cdc/default/default/upstream/7365807920150637154 {"id":7365807920150637154,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7488.out cli changefeed create --start-ts=449573240961761281 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-sink-retry-test-10091?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error-1 Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7365807920150637154,"namespace":"default","id":"changefeed-error-1","sink_uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-10091?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:57:42.533361102+08:00","start_ts":449573240961761281,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573240961761281,"checkpoint_ts":449573240961761281,"checkpoint_time":"2024-05-06 16:56:52.995"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 29-th check, retry later check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 695 0 --:--:-- --:--:-- --:--:-- 693 100 86 100 86 0 0 694 0 --:--:-- --:--:-- --:--:-- 693 + info='{"state":"normal","resolved_ts":449573240961761281,"checkpoint_ts":449573240961761281}' + echo '{"state":"normal","resolved_ts":449573240961761281,"checkpoint_ts":449573240961761281}' {"state":"normal","resolved_ts":449573240961761281,"checkpoint_ts":449573240961761281} ++ echo '{"state":"normal","resolved_ts":449573240961761281,"checkpoint_ts":449573240961761281}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 1-th time, retry later Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:57:45 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4cbc889b-7e7b-4c88-9272-3076857cba5a {"id":"4cbc889b-7e7b-4c88-9272-3076857cba5a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985864} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f4e2acf 4cbc889b-7e7b-4c88-9272-3076857cba5a /tidb/cdc/default/default/upstream/7365808138949665835 {"id":7365808138949665835,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4cbc889b-7e7b-4c88-9272-3076857cba5a {"id":"4cbc889b-7e7b-4c88-9272-3076857cba5a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985864} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f4e2acf 4cbc889b-7e7b-4c88-9272-3076857cba5a /tidb/cdc/default/default/upstream/7365808138949665835 {"id":7365808138949665835,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4cbc889b-7e7b-4c88-9272-3076857cba5a {"id":"4cbc889b-7e7b-4c88-9272-3076857cba5a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985864} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f4e2acf 4cbc889b-7e7b-4c88-9272-3076857cba5a /tidb/cdc/default/default/upstream/7365808138949665835 {"id":7365808138949665835,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:57:45 CST 2024] <<<<<< START pulsar cluster in canal_json_content_compatible case >>>>>>' [Mon May 6 16:57:45 CST 2024] <<<<<< START pulsar cluster in canal_json_content_compatible case >>>>>> + workdir=/tmp/tidb_cdc_test/canal_json_content_compatible + cluster_type=normal + cd /tmp/tidb_cdc_test/canal_json_content_compatible + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/canal_json_content_compatible/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/canal_json_content_compatible/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/canal_json_content_compatible/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/canal_json_content_compatible/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/canal_json_content_compatible/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/canal_json_content_compatible/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/canal_json_content_compatible/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/canal_json_content_compatible/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/canal_json_content_compatible/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/canal_json_content_compatible/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/canal_json_content_compatible/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/canal_json_content_compatible/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:57:45 CST 2024] <<<<<< START pulsar cluster in normal mode in canal_json_content_compatible case >>>>>>' [Mon May 6 16:57:45 CST 2024] <<<<<< START pulsar cluster in normal mode in canal_json_content_compatible case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/canal_json_content_compatible/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/canal_json_content_compatible/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/canal_json_content_compatible/pulsar-bookie Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 27.04 secs (135635604 bytes/sec) [Pipeline] { [Pipeline] cache + i=1 + '[' 1 -gt 20 ']' + sleep 2 table ddl_manager.finish_mark not exists for 30-th check, retry later check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 253 100 253 0 0 2291 0 --:--:-- --:--:-- --:--:-- 2300 + info='{"state":"warning","resolved_ts":449573241354977288,"checkpoint_ts":449573241354977288,"last_warning":{"time":"2024-05-06T16:57:44.701567176+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":449573241354977288,"checkpoint_ts":449573241354977288,"last_warning":{"time":"2024-05-06T16:57:44.701567176+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":449573241354977288,"checkpoint_ts":449573241354977288,"last_warning":{"time":"2024-05-06T16:57:44.701567176+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ jq -r .state ++ echo '{"state":"warning","resolved_ts":449573241354977288,"checkpoint_ts":449573241354977288,"last_warning":{"time":"2024-05-06T16:57:44.701567176+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ jq -r .last_warning.message ++ echo '{"state":"warning","resolved_ts":449573241354977288,"checkpoint_ts":449573241354977288,"last_warning":{"time":"2024-05-06T16:57:44.701567176+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7551.out cli changefeed remove -c changefeed-error-1 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 449573241354977288 SinkURI: pulsar://127.0.0.1:6650/ticdc-sink-retry-test-10091?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table ddl_manager.finish_mark not exists for 31-th check, retry later Starting Upstream TiDB... + set +x Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 32-th check, retry later wait process cdc.test exit for 2-th time... + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 16:57:50 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.76207622.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 33-th check, retry later public/default ++ date + echo '[Mon May 6 16:57:52 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:57:52 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_content_compatible.cli.7692.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-canal-json-content-compatible?protocol=canal-json&enable-tidb-extension=true' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: d99887c4-83c4-45f9-bdc5-065eb9c1554b Info: {"upstream_id":7365808138949665835,"namespace":"default","id":"d99887c4-83c4-45f9-bdc5-065eb9c1554b","sink_uri":"pulsar://127.0.0.1:6650/ticdc-canal-json-content-compatible?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:57:53.156650635+08:00","start_ts":449573256564834306,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573256564834306,"checkpoint_ts":449573256564834306,"checkpoint_time":"2024-05-06 16:57:52.516"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:57:53 GMT < Content-Length: 884 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b8841a93-aa4b-4d4a-a16f-44c68a4199e9 {"id":"b8841a93-aa4b-4d4a-a16f-44c68a4199e9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985870} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e8c8719 b8841a93-aa4b-4d4a-a16f-44c68a4199e9 /tidb/cdc/default/default/upstream/7365807920150637154 {"id":7365807920150637154,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b8841a93-aa4b-4d4a-a16f-44c68a4199e9 {"id":"b8841a93-aa4b-4d4a-a16f-44c68a4199e9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985870} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e8c8719 b8841a93-aa4b-4d4a-a16f-44c68a4199e9 /tidb/cdc/default/default/upstream/7365807920150637154 {"id":7365807920150637154,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b8841a93-aa4b-4d4a-a16f-44c68a4199e9 {"id":"b8841a93-aa4b-4d4a-a16f-44c68a4199e9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985870} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e8c8719 b8841a93-aa4b-4d4a-a16f-44c68a4199e9 /tidb/cdc/default/default/upstream/7365807920150637154 {"id":7365807920150637154,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7670.out cli changefeed create --start-ts=449573240961761281 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-sink-retry-test-10091?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error-2 Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7365807920150637154,"namespace":"default","id":"changefeed-error-2","sink_uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-10091?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:57:53.900288515+08:00","start_ts":449573240961761281,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573240961761281,"checkpoint_ts":449573240961761281,"checkpoint_time":"2024-05-06 16:56:52.995"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 34-th check, retry later + set +x + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449573240961761281, "checkpoint_time": "2024-05-06 16:56:52.995", "error": { "time": "2024-05-06T16:57:53.971640402+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449573240961761281 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449573240961761281, "checkpoint_time": "2024-05-06 16:56:52.995", "error": { "time": "2024-05-06T16:57:53.971640402+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449573240961761281 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449573240961761281, "checkpoint_time": "2024-05-06 16:56:52.995", "error": { "time": "2024-05-06T16:57:53.971640402+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449573240961761281 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365807920150637154, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449573240961761281, '"checkpoint_time":' '"2024-05-06' '16:56:52.995",' '"error":' '{' '"time":' '"2024-05-06T16:57:53.971640402+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449573240961761281 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365807920150637154, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449573240961761281, '"checkpoint_time":' '"2024-05-06' '16:56:52.995",' '"error":' '{' '"time":' '"2024-05-06T16:57:53.971640402+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449573240961761281 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449573240961761281 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449573240961761281 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7746.out cli changefeed remove -c changefeed-error-2 table ddl_manager.finish_mark not exists for 35-th check, retry later Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 449573240961761281 SinkURI: pulsar://127.0.0.1:6650/ticdc-sink-retry-test-10091?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347eece40008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq, pid:7506, start at 2024-05-06 16:57:56.298891129 +0800 CST m=+7.430911625 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:56.318 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:56.332 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:56.332 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table ddl_manager.finish_mark not exists for 36-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347eece40008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq, pid:7506, start at 2024-05-06 16:57:56.298891129 +0800 CST m=+7.430911625 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:56.318 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:56.332 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:56.332 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347eedc0000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq, pid:7543, start at 2024-05-06 16:57:56.371487139 +0800 CST m=+7.403310656 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:56.388 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:56.385 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:56.385 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 3-th time... table ddl_manager.finish_mark not exists for 37-th check, retry later [Mon May 6 16:57:59 CST 2024] <<<<<< START Pulsar consumer in canal_json_content_compatible case >>>>>> cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:57:59 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.77997801.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 38-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:03 GMT < Content-Length: 884 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/caee8c1e-d6a6-42cf-9fd3-5b6b3597ef78 {"id":"caee8c1e-d6a6-42cf-9fd3-5b6b3597ef78","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985880} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e8c8774 caee8c1e-d6a6-42cf-9fd3-5b6b3597ef78 /tidb/cdc/default/default/upstream/7365807920150637154 {"id":7365807920150637154,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/caee8c1e-d6a6-42cf-9fd3-5b6b3597ef78 {"id":"caee8c1e-d6a6-42cf-9fd3-5b6b3597ef78","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985880} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e8c8774 caee8c1e-d6a6-42cf-9fd3-5b6b3597ef78 /tidb/cdc/default/default/upstream/7365807920150637154 {"id":7365807920150637154,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/caee8c1e-d6a6-42cf-9fd3-5b6b3597ef78 {"id":"caee8c1e-d6a6-42cf-9fd3-5b6b3597ef78","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985880} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e8c8774 caee8c1e-d6a6-42cf-9fd3-5b6b3597ef78 /tidb/cdc/default/default/upstream/7365807920150637154 {"id":7365807920150637154,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7858.out cli changefeed create --start-ts=0 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-sink-retry-test-10091?protocol=canal-json&enable-tidb-extension=true' -c changefeed-initialize-error Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7365807920150637154,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-10091?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:58:03.557959973+08:00","start_ts":449573259416436742,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573259416436742,"checkpoint_ts":449573259416436742,"checkpoint_time":"2024-05-06 16:58:03.394"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449573259416436742, "checkpoint_time": "2024-05-06 16:58:03.394", "error": { "time": "2024-05-06T16:58:03.734235242+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449573259416436742, "checkpoint_time": "2024-05-06 16:58:03.394", "error": { "time": "2024-05-06T16:58:03.734235242+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449573259416436742, "checkpoint_time": "2024-05-06 16:58:03.394", "error": { "time": "2024-05-06T16:58:03.734235242+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365807920150637154, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449573259416436742, '"checkpoint_time":' '"2024-05-06' '16:58:03.394",' '"error":' '{' '"time":' '"2024-05-06T16:58:03.734235242+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 39-th check, retry later table test.finish_mark not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 40-th check, retry later ++ date + echo '[Mon May 6 16:58:05 CST 2024] <<<<<< START pulsar cluster in processor_stop_delay case >>>>>>' [Mon May 6 16:58:05 CST 2024] <<<<<< START pulsar cluster in processor_stop_delay case >>>>>> + workdir=/tmp/tidb_cdc_test/processor_stop_delay + cluster_type=normal + cd /tmp/tidb_cdc_test/processor_stop_delay + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/processor_stop_delay/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/processor_stop_delay/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/processor_stop_delay/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_stop_delay/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/processor_stop_delay/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/processor_stop_delay/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/processor_stop_delay/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/processor_stop_delay/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_stop_delay/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/processor_stop_delay/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/processor_stop_delay/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/processor_stop_delay/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:58:05 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_stop_delay case >>>>>>' [Mon May 6 16:58:05 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_stop_delay case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/processor_stop_delay/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/processor_stop_delay/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/processor_stop_delay/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449573259416436742, "checkpoint_time": "2024-05-06 16:58:03.394", "error": { "time": "2024-05-06T16:58:03.734235242+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449573259416436742, "checkpoint_time": "2024-05-06 16:58:03.394", "error": { "time": "2024-05-06T16:58:03.734235242+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449573259416436742, "checkpoint_time": "2024-05-06 16:58:03.394", "error": { "time": "2024-05-06T16:58:03.734235242+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365807920150637154, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449573259416436742, '"checkpoint_time":' '"2024-05-06' '16:58:03.394",' '"error":' '{' '"time":' '"2024-05-06T16:58:03.734235242+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later table test.finish_mark not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 41-th check, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table test.finish_mark not exists for 3-th check, retry later table ddl_manager.finish_mark not exists for 42-th check, retry later + nc -z localhost 6650 + i=3 + '[' 3 -gt 20 ']' + sleep 2 check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449573261015777291, "checkpoint_time": "2024-05-06 16:58:09.495", "error": null }' + echo '{ "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449573261015777291, "checkpoint_time": "2024-05-06 16:58:09.495", "error": null }' { "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449573261015777291, "checkpoint_time": "2024-05-06 16:58:09.495", "error": null } ++ echo '{' '"upstream_id":' 7365807920150637154, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449573261015777291, '"checkpoint_time":' '"2024-05-06' '16:58:09.495",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365807920150637154, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449573261015777291, '"checkpoint_time":' '"2024-05-06' '16:58:09.495",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.8023.out cli changefeed pause -c changefeed-initialize-error table test.finish_mark not exists for 4-th check, retry later PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 43-th check, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449573261277921291, "checkpoint_time": "2024-05-06 16:58:10.495", "error": { "time": "2024-05-06T16:58:03.734235242+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449573261277921291, "checkpoint_time": "2024-05-06 16:58:10.495", "error": { "time": "2024-05-06T16:58:03.734235242+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449573261277921291, "checkpoint_time": "2024-05-06 16:58:10.495", "error": { "time": "2024-05-06T16:58:03.734235242+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365807920150637154, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449573261277921291, '"checkpoint_time":' '"2024-05-06' '16:58:10.495",' '"error":' '{' '"time":' '"2024-05-06T16:58:03.734235242+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365807920150637154, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449573261277921291, '"checkpoint_time":' '"2024-05-06' '16:58:10.495",' '"error":' '{' '"time":' '"2024-05-06T16:58:03.734235242+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.8119.out cli changefeed resume -c changefeed-initialize-error table test.finish_mark not exists for 5-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 25.41 secs (144354531 bytes/sec) PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... [Pipeline] { [Pipeline] cache table ddl_manager.finish_mark not exists for 44-th check, retry later table test.finish_mark not exists for 6-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449573261277921291, "checkpoint_time": "2024-05-06 16:58:10.495", "error": null }' + echo '{ "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449573261277921291, "checkpoint_time": "2024-05-06 16:58:10.495", "error": null }' { "upstream_id": 7365807920150637154, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449573261277921291, "checkpoint_time": "2024-05-06 16:58:10.495", "error": null } ++ echo '{' '"upstream_id":' 7365807920150637154, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449573261277921291, '"checkpoint_time":' '"2024-05-06' '16:58:10.495",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365807920150637154, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449573261277921291, '"checkpoint_time":' '"2024-05-06' '16:58:10.495",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.8196.out cli changefeed remove -c changefeed-initialize-error table ddl_manager.finish_mark not exists for 45-th check, retry later Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 449573261277921291 SinkURI: pulsar://127.0.0.1:6650/ticdc-sink-retry-test-10091?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 7-th check, retry later + set +x table ddl_manager.finish_mark not exists for 46-th check, retry later wait process cdc.test exit for 1-th time... Tenant not found Reason: Tenant not found + i=1 + '[' 1 -gt 20 ']' + sleep 2 wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 16:58:19 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> table test.finish_mark not exists for 8-th check, retry later table ddl_manager.finish_mark not exists for 47-th check, retry later + /usr/local/pulsar/bin/pulsar-admin namespaces list public table test.finish_mark not exists for 9-th check, retry later table ddl_manager.finish_mark not exists for 48-th check, retry later table test.finish_mark not exists for 10-th check, retry later public/default ++ date + echo '[Mon May 6 16:58:23 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:58:23 CST 2024] <<<<<< pulsar is ready >>>>>> [Mon May 6 16:58:23 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.93799381.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo '' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 49-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc) 3667965440 bytes in 9.59 secs (382390316 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh table ddl_manager.finish_mark not exists for 50-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G09 Run cases: gc_safepoint changefeed_pause_resume cli_with_auth savepoint synced_status PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=5c881b01-f1b6-4c55-914d-19ed88aec6ad BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-q6kw-vxk5b HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-q6kwb GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-q6kwb pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-q6kw-vxk5b GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-q6kw-vxk5b GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: pulsar... <<================= [Pipeline] sh The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G17 Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=6b9428f8-90f0-49c9-9d66-804613bd271c BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-rjrh-lnz8j HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-rjrhn GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-rjrhn pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-rjrh-lnz8j GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-rjrh-lnz8j GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: pulsar... <<================= [Pipeline] sh table test.finish_mark not exists for 11-th check, retry later [Mon May 6 16:58:26 CST 2024] <<<<<< skip test case clustered_index for pulsar! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G10 [Pipeline] sh + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:26 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/543175b9-a461-485a-b55d-8ab33db52df2 {"id":"543175b9-a461-485a-b55d-8ab33db52df2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985904} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f89f542 543175b9-a461-485a-b55d-8ab33db52df2 /tidb/cdc/default/default/upstream/7365808199691015400 {"id":7365808199691015400,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/543175b9-a461-485a-b55d-8ab33db52df2 {"id":"543175b9-a461-485a-b55d-8ab33db52df2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985904} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f89f542 543175b9-a461-485a-b55d-8ab33db52df2 /tidb/cdc/default/default/upstream/7365808199691015400 {"id":7365808199691015400,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/543175b9-a461-485a-b55d-8ab33db52df2 {"id":"543175b9-a461-485a-b55d-8ab33db52df2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985904} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f89f542 543175b9-a461-485a-b55d-8ab33db52df2 /tidb/cdc/default/default/upstream/7365808199691015400 {"id":7365808199691015400,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G08 Run cases: processor_err_chan changefeed_reconstruct multi_capture synced_status_with_redo PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=7a722c50-6084-4262-a0a3-a94e0d58e040 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-j6m9-tmj32 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-j6m97 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-j6m9-tmj32 pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-j6m97 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-j6m9-tmj32 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh Run cases: default_value simple cdc_server_tips event_filter sql_mode PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=27d164d8-46ea-4aaf-a02f-3adbc9403dfd BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7txd-3l8p2 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-7txd9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-7txd9 pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7txd-3l8p2 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7txd-3l8p2 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: pulsar... <<================= + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=88a018bf-61fa-4042-8a7f-af6065eefe8d BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0c9z-7hn20 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-0c9zg GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-0c9zg pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0c9z-7hn20 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0c9z-7hn20 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: pulsar... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G14 table ddl_manager.finish_mark not exists for 51-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G13 Run cases: tiflash region_merge common_1 PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=00651d38-5654-46dc-b0d5-2170f253c502 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-x5q3-r7q9p HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-x5q3s GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-x5q3s pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-x5q3-r7q9p GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-x5q3-r7q9p GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory [Pipeline] sh Run cases: changefeed_finish force_replicate_table PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=3ba4d875-f0cd-4e7d-8a9f-9a3854a8383d BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-19s0-56793 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-19s0g GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-19s0g pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-19s0-56793 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-19s0-56793 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G16 [Pipeline] sh table test.finish_mark not exists for 12-th check, retry later [Mon May 6 16:58:28 CST 2024] <<<<<< START Pulsar consumer in processor_stop_delay case >>>>>> Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=c0188fe8-64e8-4415-8440-39d295b57003 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-1bd3-dwfrg HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-1bd30 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-1bd3-dwfrg pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-1bd30 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-1bd3-dwfrg GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G15 Run cases: new_ci_collation batch_add_table multi_rocks PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=2d780931-7708-480b-9272-be76cc327586 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-5fm7-zn46k HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-5fm70 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-5fm7-zn46k pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-5fm70 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-5fm7-zn46k GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: pulsar... <<================= table processor_stop_delay.t not exists for 1-th check, retry later [Pipeline] // timeout [Pipeline] } [Pipeline] // timeout [Pipeline] } + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G12 Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=953cd3fe-8c4f-498b-b548-e5d261e32152 JENKINS_NODE_COOKIE=281e363f-9ed9-4b73-9bc2-d3fc851d0eeb BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7wx1-4ndrn HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1787402136024256514","prowjobid":"953cd3fe-8c4f-498b-b548-e5d261e32152","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1555 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256514 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1555/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-7wx1w GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7wx1-4ndrn pingcap_tiflow_pull_cdc_integration_pulsar_test_1555-7wx1w GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7wx1-4ndrn GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1555 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: pulsar... <<================= table ddl_manager.finish_mark not exists for 52-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table test.finish_mark not exists for 13-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table processor_stop_delay.t exists check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 53-th check, retry later Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... The 1 times to try to start tidb cluster... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table test.finish_mark not exists for 14-th check, retry later check diff failed 2-th time, retry later Verifying downstream PD is started... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... table test.finish_mark not exists for 15-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 54-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff successfully Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 55-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... table test.finish_mark not exists for 16-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 56-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark exists check diff successfully Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc already exists) start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 57-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 58-th check, retry later check diff failed 3-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3481b2a00012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-q6kw-vxk5b, pid:1386, start at 2024-05-06 16:58:41.728439021 +0800 CST m=+5.821236195 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:41.738 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:41.704 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:41.704 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3481c1f40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-j6m9-tmj32, pid:1402, start at 2024-05-06 16:58:42.722232257 +0800 CST m=+5.600973340 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:42.730 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:42.735 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:42.735 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3481c8700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-x5q3-r7q9p, pid:1313, start at 2024-05-06 16:58:43.150256129 +0800 CST m=+5.401892678 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:43.158 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:43.150 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:43.150 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3481a7b8000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-rjrh-lnz8j, pid:1265, start at 2024-05-06 16:58:41.018117609 +0800 CST m=+5.357635118 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:41.027 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:41.006 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:41.006 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3481a7b8000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-rjrh-lnz8j, pid:1265, start at 2024-05-06 16:58:41.018117609 +0800 CST m=+5.357635118 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:41.027 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:41.006 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:41.006 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3481a86c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-rjrh-lnz8j, pid:1320, start at 2024-05-06 16:58:41.07596152 +0800 CST m=+5.351646708 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:41.084 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:41.051 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:41.051 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 59-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3481b2a00012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-q6kw-vxk5b, pid:1386, start at 2024-05-06 16:58:41.728439021 +0800 CST m=+5.821236195 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:41.738 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:41.704 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:41.704 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3481b3680002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-q6kw-vxk5b, pid:1454, start at 2024-05-06 16:58:41.757681101 +0800 CST m=+5.779938064 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:41.766 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:41.754 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:41.754 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3481cd880006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-19s0-56793, pid:1400, start at 2024-05-06 16:58:43.43489572 +0800 CST m=+5.460422489 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:43.445 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:43.426 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:43.426 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3481c1f40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-j6m9-tmj32, pid:1402, start at 2024-05-06 16:58:42.722232257 +0800 CST m=+5.600973340 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:42.730 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:42.735 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:42.735 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3481c1e40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-j6m9-tmj32, pid:1487, start at 2024-05-06 16:58:42.718505474 +0800 CST m=+5.531577705 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:42.727 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:42.731 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:42.731 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table test.finish_mark not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // stage [Pipeline] } [Pipeline] // stage Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3481c8700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-x5q3-r7q9p, pid:1313, start at 2024-05-06 16:58:43.150256129 +0800 CST m=+5.401892678 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:43.158 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:43.150 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:43.150 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3481caa40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-x5q3-r7q9p, pid:1393, start at 2024-05-06 16:58:43.290677231 +0800 CST m=+5.473135637 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:43.298 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:43.291 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:43.291 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 16:58:45 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=return(true);github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.27632765.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 60-th check, retry later check diff failed 5-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3481cd880006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-19s0-56793, pid:1400, start at 2024-05-06 16:58:43.43489572 +0800 CST m=+5.460422489 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:43.445 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:43.426 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:43.426 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3481cf280008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-19s0-56793, pid:1481, start at 2024-05-06 16:58:43.539946895 +0800 CST m=+5.498623435 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:43.548 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:43.530 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:43.530 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ++ date + echo '[Mon May 6 16:58:46 CST 2024] <<<<<< START pulsar cluster in gc_safepoint case >>>>>>' [Mon May 6 16:58:46 CST 2024] <<<<<< START pulsar cluster in gc_safepoint case >>>>>> + workdir=/tmp/tidb_cdc_test/gc_safepoint + cluster_type=normal + cd /tmp/tidb_cdc_test/gc_safepoint + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/gc_safepoint/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/gc_safepoint/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/gc_safepoint/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/gc_safepoint/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/gc_safepoint/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/gc_safepoint/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/gc_safepoint/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/gc_safepoint/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/gc_safepoint/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/gc_safepoint/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/gc_safepoint/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/gc_safepoint/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:58:46 CST 2024] <<<<<< START pulsar cluster in normal mode in gc_safepoint case >>>>>>' [Mon May 6 16:58:46 CST 2024] <<<<<< START pulsar cluster in normal mode in gc_safepoint case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/gc_safepoint/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/gc_safepoint/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/gc_safepoint/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34820c8c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0c9z-7hn20, pid:1465, start at 2024-05-06 16:58:47.475899952 +0800 CST m=+5.740759286 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:47.485 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:47.459 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:47.459 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ date + echo '[Mon May 6 16:58:47 CST 2024] <<<<<< START pulsar cluster in processor_err_chan case >>>>>>' [Mon May 6 16:58:47 CST 2024] <<<<<< START pulsar cluster in processor_err_chan case >>>>>> + workdir=/tmp/tidb_cdc_test/processor_err_chan + cluster_type=normal + cd /tmp/tidb_cdc_test/processor_err_chan + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/processor_err_chan/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/processor_err_chan/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/processor_err_chan/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_err_chan/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/processor_err_chan/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/processor_err_chan/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/processor_err_chan/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/processor_err_chan/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_err_chan/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/processor_err_chan/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/processor_err_chan/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/processor_err_chan/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:58:47 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_err_chan case >>>>>>' [Mon May 6 16:58:47 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_err_chan case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/processor_err_chan/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/processor_err_chan/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/processor_err_chan/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 table ddl_manager.finish_mark not exists for 61-th check, retry later check diff failed 6-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2764.out cli tso query --pd=http://127.0.0.1:2379 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ date + echo '[Mon May 6 16:58:48 CST 2024] <<<<<< START pulsar cluster in changefeed_finish case >>>>>>' [Mon May 6 16:58:48 CST 2024] <<<<<< START pulsar cluster in changefeed_finish case >>>>>> + workdir=/tmp/tidb_cdc_test/changefeed_finish + cluster_type=normal + cd /tmp/tidb_cdc_test/changefeed_finish + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_finish/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/changefeed_finish/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/changefeed_finish/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_finish/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/changefeed_finish/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/changefeed_finish/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/changefeed_finish/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/changefeed_finish/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_finish/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/changefeed_finish/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/changefeed_finish/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/changefeed_finish/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:58:48 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_finish case >>>>>>' [Mon May 6 16:58:48 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_finish case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/changefeed_finish/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/changefeed_finish/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/changefeed_finish/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 [Pipeline] } table test.finish_mark not exists for 4-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34820e1c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7wx1-4ndrn, pid:1486, start at 2024-05-06 16:58:47.591638885 +0800 CST m=+5.625175772 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:47.599 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:47.559 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:47.559 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34820e1c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7wx1-4ndrn, pid:1486, start at 2024-05-06 16:58:47.591638885 +0800 CST m=+5.625175772 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:47.599 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:47.559 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:47.559 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34820fbc0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7wx1-4ndrn, pid:1566, start at 2024-05-06 16:58:47.668921644 +0800 CST m=+5.648999821 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:47.676 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:47.663 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:47.663 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Pipeline] // container [Pipeline] } Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:49 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc0de295-8c77-4c9d-8d6f-d7696ad0a76b {"id":"cc0de295-8c77-4c9d-8d6f-d7696ad0a76b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985926} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d203f21cb cc0de295-8c77-4c9d-8d6f-d7696ad0a76b /tidb/cdc/default/default/upstream/7365808405637481761 {"id":7365808405637481761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc0de295-8c77-4c9d-8d6f-d7696ad0a76b {"id":"cc0de295-8c77-4c9d-8d6f-d7696ad0a76b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985926} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d203f21cb cc0de295-8c77-4c9d-8d6f-d7696ad0a76b /tidb/cdc/default/default/upstream/7365808405637481761 {"id":7365808405637481761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc0de295-8c77-4c9d-8d6f-d7696ad0a76b {"id":"cc0de295-8c77-4c9d-8d6f-d7696ad0a76b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985926} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d203f21cb cc0de295-8c77-4c9d-8d6f-d7696ad0a76b /tidb/cdc/default/default/upstream/7365808405637481761 {"id":7365808405637481761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:58:49 CST 2024] <<<<<< START pulsar cluster in processor_resolved_ts_fallback case >>>>>>' [Mon May 6 16:58:49 CST 2024] <<<<<< START pulsar cluster in processor_resolved_ts_fallback case >>>>>> + workdir=/tmp/tidb_cdc_test/processor_resolved_ts_fallback + cluster_type=normal + cd /tmp/tidb_cdc_test/processor_resolved_ts_fallback + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/processor_resolved_ts_fallback/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/processor_resolved_ts_fallback/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/processor_resolved_ts_fallback/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/processor_resolved_ts_fallback/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/processor_resolved_ts_fallback/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/processor_resolved_ts_fallback/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:58:49 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_resolved_ts_fallback case >>>>>>' [Mon May 6 16:58:49 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_resolved_ts_fallback case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/processor_resolved_ts_fallback/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 [Pipeline] // container [Pipeline] } + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // dir table ddl_manager.finish_mark not exists for 62-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348228800010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-1bd3-dwfrg, pid:1341, start at 2024-05-06 16:58:49.291621678 +0800 CST m=+10.046996742 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:49.308 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:49.298 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:49.298 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // withEnv [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34820c8c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0c9z-7hn20, pid:1465, start at 2024-05-06 16:58:47.475899952 +0800 CST m=+5.740759286 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:47.485 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:47.459 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:47.459 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348211880008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0c9z-7hn20, pid:1521, start at 2024-05-06 16:58:47.789036813 +0800 CST m=+5.993072250 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:47.796 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:47.778 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:47.778 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='449573271227858945 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573271227858945 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:58:49 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.27992801.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // timeout [Pipeline] } + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public [Pipeline] // node check diff successfully + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table test.finish_mark not exists for 5-th check, retry later wait process cdc.test exit for 1-th time... [Pipeline] } ++ date + echo '[Mon May 6 16:58:51 CST 2024] <<<<<< START pulsar cluster in capture_session_done_during_task case >>>>>>' [Mon May 6 16:58:51 CST 2024] <<<<<< START pulsar cluster in capture_session_done_during_task case >>>>>> + workdir=/tmp/tidb_cdc_test/capture_session_done_during_task + cluster_type=normal + cd /tmp/tidb_cdc_test/capture_session_done_during_task + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/capture_session_done_during_task/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/capture_session_done_during_task/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/capture_session_done_during_task/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/capture_session_done_during_task/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/capture_session_done_during_task/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/capture_session_done_during_task/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/capture_session_done_during_task/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:58:51 CST 2024] <<<<<< START pulsar cluster in normal mode in capture_session_done_during_task case >>>>>>' [Mon May 6 16:58:51 CST 2024] <<<<<< START pulsar cluster in normal mode in capture_session_done_during_task case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/capture_session_done_during_task/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/capture_session_done_during_task/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/capture_session_done_during_task/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 [Pipeline] // node wait process cdc.test exit for 2-th time... [Pipeline] } + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34822004000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-5fm7-zn46k, pid:1493, start at 2024-05-06 16:58:48.719631123 +0800 CST m=+5.460233748 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:48.727 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:48.705 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:48.705 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34822004000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-5fm7-zn46k, pid:1493, start at 2024-05-06 16:58:48.719631123 +0800 CST m=+5.460233748 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:48.727 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:48.705 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:48.705 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348220c80010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-5fm7-zn46k, pid:1567, start at 2024-05-06 16:58:48.771741317 +0800 CST m=+5.453943062 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:48.777 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:48.754 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:48.754 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table ddl_manager.finish_mark not exists for 63-th check, retry later [Pipeline] // podTemplate [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348228800010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-1bd3-dwfrg, pid:1341, start at 2024-05-06 16:58:49.291621678 +0800 CST m=+10.046996742 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:49.308 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:49.298 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:49.298 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348228740005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-1bd3-dwfrg, pid:1377, start at 2024-05-06 16:58:49.27131058 +0800 CST m=+9.857836714 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:49.287 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:49.296 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:49.296 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_resign/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_resign/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // podTemplate ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] } [Pipeline] // container + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2845.out cli tso query --pd=http://127.0.0.1:2379 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:58:52 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> [Pipeline] } [Pipeline] // withEnv [Pipeline] } + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table test.finish_mark not exists for 6-th check, retry later [Pipeline] // withEnv [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Tenant not found Reason: Tenant not found + i=1 + '[' 1 -gt 20 ']' + sleep 2 < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:53 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/85b76a08-241f-4966-adf2-87c83bb84754 {"id":"85b76a08-241f-4966-adf2-87c83bb84754","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985930} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20455cd2 85b76a08-241f-4966-adf2-87c83bb84754 /tidb/cdc/default/default/upstream/7365808414137410017 {"id":7365808414137410017,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/85b76a08-241f-4966-adf2-87c83bb84754 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... {"id":"85b76a08-241f-4966-adf2-87c83bb84754","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985930} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public /tidb/cdc/default/__cdc_meta__/owner/22318f4d20455cd2 85b76a08-241f-4966-adf2-87c83bb84754 /tidb/cdc/default/default/upstream/7365808414137410017 {"id":7365808414137410017,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/85b76a08-241f-4966-adf2-87c83bb84754 {"id":"85b76a08-241f-4966-adf2-87c83bb84754","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985930} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20455cd2 85b76a08-241f-4966-adf2-87c83bb84754 /tidb/cdc/default/default/upstream/7365808414137410017 {"id":7365808414137410017,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:58:53 CST 2024] <<<<<< START pulsar cluster in tiflash case >>>>>>' [Mon May 6 16:58:53 CST 2024] <<<<<< START pulsar cluster in tiflash case >>>>>> + workdir=/tmp/tidb_cdc_test/tiflash + cluster_type=mtls + cd /tmp/tidb_cdc_test/tiflash + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/tiflash/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/tiflash/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/tiflash/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/tiflash/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/tiflash/pulsar_standalone.conf + pulsar_port=6650 + '[' mtls == mtls ']' + pulsar_port=6651 + gen_mtls_config + openssl genrsa -out ca.key.pem 2048 Generating RSA private key, 2048 bit long modulus ....................................................................+++ .........................................................................+++ e is 65537 (0x10001) + openssl req -x509 -new -nodes -key ca.key.pem -subj /CN=CARoot -days 365 -out ca.cert.pem + openssl genrsa -out server.key.pem 2048 Generating RSA private key, 2048 bit long modulus [Pipeline] // stage [Pipeline] } [Pipeline] // node [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34825e4c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7txd-3l8p2, pid:1390, start at 2024-05-06 16:58:52.714069852 +0800 CST m=+10.060962636 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:52.736 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:52.741 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:52.741 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) .......................................................................+++ ..........................................................................................................+++ e is 65537 (0x10001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in server.key.pem -out server.key-pk8.pem -nocrypt + echo '[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + openssl req -new -config server.conf -key server.key.pem -out server.csr.pem -sha256 + openssl x509 -req -in server.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out server.cert.pem -days 365 -extensions v3_ext -extfile server.conf -sha256 Signature ok subject=/CN=server Getting CA Private Key + openssl genrsa -out broker_client.key.pem 2048 Generating RSA private key, 2048 bit long modulus ..................................+++ ..............+++ e is 65537 (0x10001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in broker_client.key.pem -out broker_client.key-pk8.pem -nocrypt + openssl req -new -subj /CN=broker_client -key broker_client.key.pem -out broker_client.csr.pem -sha256 + openssl x509 -req -in broker_client.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out broker_client.cert.pem -days 365 -sha256 Signature ok subject=/CN=broker_client Getting CA Private Key + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/tiflash/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/tiflash/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' + echo ' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem' ++ date + echo '[Mon May 6 16:58:53 CST 2024] <<<<<< START pulsar cluster in mtls mode in tiflash case >>>>>>' [Mon May 6 16:58:53 CST 2024] <<<<<< START pulsar cluster in mtls mode in tiflash case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6651 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/tiflash/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/tiflash/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/tiflash/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 [Pipeline] // podTemplate [Pipeline] } + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 [Pipeline] // withEnv public/default [Pipeline] } [Pipeline] // stage [Pipeline] } ++ date + echo '[Mon May 6 16:58:53 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:58:53 CST 2024] <<<<<< pulsar is ready >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3482557c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8, pid:9335, start at 2024-05-06 16:58:52.15240526 +0800 CST m=+7.649301848 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:52.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:52.127 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:52.127 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3482557c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8, pid:9335, start at 2024-05-06 16:58:52.15240526 +0800 CST m=+7.649301848 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:52.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:52.127 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:52.127 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348250a8000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8, pid:9385, start at 2024-05-06 16:58:51.833379688 +0800 CST m=+7.272319046 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:51.845 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:51.818 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:51.818 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... + set +x + tso='449573272278532097 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573272278532097 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:58:53 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.28752877.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 64-th check, retry later + /usr/local/pulsar/bin/pulsar-admin namespaces list public Tenant not found Reason: Tenant not found + i=1 + '[' 1 -gt 20 ']' + sleep 2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34825e4c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7txd-3l8p2, pid:1390, start at 2024-05-06 16:58:52.714069852 +0800 CST m=+10.060962636 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:52.736 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:52.741 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:52.741 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34825d44000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7txd-3l8p2, pid:1424, start at 2024-05-06 16:58:52.657585478 +0800 CST m=+9.851987450 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:52.689 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:52.677 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:52.677 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... [Mon May 6 16:58:55 CST 2024] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.29172919.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/simple/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/simple/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/simple/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ++ date + echo '[Mon May 6 16:58:55 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:58:55 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.cli.3238.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-15533?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 + nc -z localhost 6651 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.10742.out cli tso query --pd=http://127.0.0.1:2379 + nc -z localhost 6650 + i=3 + '[' 3 -gt 20 ']' + sleep 2 [Mon May 6 16:58:56 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.27022704.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + grep -q 'failed to get info:' + echo '' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 65-th check, retry later [Mon May 6 16:58:56 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/ProcessorAddTableError=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.33703372.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: 6760f65e-620c-469e-9d8a-d332106a54ce Info: {"upstream_id":7365808405637481761,"namespace":"default","id":"6760f65e-620c-469e-9d8a-d332106a54ce","sink_uri":"pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-15533?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:58:56.421598386+08:00","start_ts":449573273142820868,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573273142820868,"checkpoint_ts":449573273142820868,"checkpoint_time":"2024-05-06 16:58:55.756"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 7-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.2605.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:57 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/99928ea0-bab3-46fd-b294-523e42917f3c {"id":"99928ea0-bab3-46fd-b294-523e42917f3c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985934} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d205190d2 99928ea0-bab3-46fd-b294-523e42917f3c /tidb/cdc/default/default/upstream/7365808427702158470 {"id":7365808427702158470,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/99928ea0-bab3-46fd-b294-523e42917f3c {"id":"99928ea0-bab3-46fd-b294-523e42917f3c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985934} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d205190d2 99928ea0-bab3-46fd-b294-523e42917f3c /tidb/cdc/default/default/upstream/7365808427702158470 {"id":7365808427702158470,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/99928ea0-bab3-46fd-b294-523e42917f3c {"id":"99928ea0-bab3-46fd-b294-523e42917f3c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985934} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d205190d2 99928ea0-bab3-46fd-b294-523e42917f3c /tidb/cdc/default/default/upstream/7365808427702158470 {"id":7365808427702158470,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:58:57 CST 2024] <<<<<< START pulsar cluster in resolve_lock case >>>>>>' [Mon May 6 16:58:57 CST 2024] <<<<<< START pulsar cluster in resolve_lock case >>>>>> + workdir=/tmp/tidb_cdc_test/resolve_lock + cluster_type=normal + cd /tmp/tidb_cdc_test/resolve_lock + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/resolve_lock/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/resolve_lock/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/resolve_lock/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/resolve_lock/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/resolve_lock/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/resolve_lock/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/resolve_lock/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/resolve_lock/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/resolve_lock/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/resolve_lock/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/resolve_lock/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/resolve_lock/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:58:57 CST 2024] <<<<<< START pulsar cluster in normal mode in resolve_lock case >>>>>>' [Mon May 6 16:58:57 CST 2024] <<<<<< START pulsar cluster in normal mode in resolve_lock case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/resolve_lock/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/resolve_lock/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/resolve_lock/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 public/default ++ date + echo '[Mon May 6 16:58:57 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:58:57 CST 2024] <<<<<< pulsar is ready >>>>>> [Mon May 6 16:58:57 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.32843286.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6651 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + set +x [Mon May 6 16:58:57 CST 2024] <<<<<< START Pulsar consumer in processor_resolved_ts_fallback case >>>>>> + set +x + tso='449573273318719489 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573273318719489 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:58:57 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.1078210784.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + nc -z localhost 6650 + i=4 + '[' 4 -gt 20 ']' + sleep 2 table ddl_manager.finish_mark not exists for 66-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:58 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/074267fd-cc9d-400f-ad73-1b1f99d91b9e {"id":"074267fd-cc9d-400f-ad73-1b1f99d91b9e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985935} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d205ade1e 074267fd-cc9d-400f-ad73-1b1f99d91b9e /tidb/cdc/default/default/upstream/7365808441496147259 {"id":7365808441496147259,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/074267fd-cc9d-400f-ad73-1b1f99d91b9e {"id":"074267fd-cc9d-400f-ad73-1b1f99d91b9e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985935} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d205ade1e 074267fd-cc9d-400f-ad73-1b1f99d91b9e /tidb/cdc/default/default/upstream/7365808441496147259 {"id":7365808441496147259,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/074267fd-cc9d-400f-ad73-1b1f99d91b9e {"id":"074267fd-cc9d-400f-ad73-1b1f99d91b9e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985935} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d205ade1e 074267fd-cc9d-400f-ad73-1b1f99d91b9e /tidb/cdc/default/default/upstream/7365808441496147259 {"id":7365808441496147259,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:58:58 CST 2024] <<<<<< START pulsar cluster in batch_add_table case >>>>>>' [Mon May 6 16:58:58 CST 2024] <<<<<< START pulsar cluster in batch_add_table case >>>>>> + workdir=/tmp/tidb_cdc_test/batch_add_table + cluster_type=normal + cd /tmp/tidb_cdc_test/batch_add_table + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/batch_add_table/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/batch_add_table/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/batch_add_table/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/batch_add_table/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/batch_add_table/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/batch_add_table/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/batch_add_table/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/batch_add_table/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/batch_add_table/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/batch_add_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/batch_add_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/batch_add_table/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:58:58 CST 2024] <<<<<< START pulsar cluster in normal mode in batch_add_table case >>>>>>' [Mon May 6 16:58:58 CST 2024] <<<<<< START pulsar cluster in normal mode in batch_add_table case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/batch_add_table/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/batch_add_table/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/batch_add_table/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 table test.finish_mark exists check diff successfully cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table processor_resolved_ts_fallback.t1 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:59 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b9ea401-a5ba-46ec-95a6-48d40631e240 {"id":"8b9ea401-a5ba-46ec-95a6-48d40631e240","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985936} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d204c92e3 8b9ea401-a5ba-46ec-95a6-48d40631e240 /tidb/cdc/default/default/upstream/7365808414652405965 {"id":7365808414652405965,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b9ea401-a5ba-46ec-95a6-48d40631e240 {"id":"8b9ea401-a5ba-46ec-95a6-48d40631e240","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985936} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d204c92e3 8b9ea401-a5ba-46ec-95a6-48d40631e240 /tidb/cdc/default/default/upstream/7365808414652405965 {"id":7365808414652405965,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b9ea401-a5ba-46ec-95a6-48d40631e240 {"id":"8b9ea401-a5ba-46ec-95a6-48d40631e240","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985936} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d204c92e3 8b9ea401-a5ba-46ec-95a6-48d40631e240 /tidb/cdc/default/default/upstream/7365808414652405965 {"id":7365808414652405965,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc cli capture list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully ++ date + echo '[Mon May 6 16:58:59 CST 2024] <<<<<< START pulsar cluster in owner_resign case >>>>>>' [Mon May 6 16:58:59 CST 2024] <<<<<< START pulsar cluster in owner_resign case >>>>>> + workdir=/tmp/tidb_cdc_test/owner_resign + cluster_type=normal + cd /tmp/tidb_cdc_test/owner_resign + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/owner_resign/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/owner_resign/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/owner_resign/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/owner_resign/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/owner_resign/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/owner_resign/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/owner_resign/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/owner_resign/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/owner_resign/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/owner_resign/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/owner_resign/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/owner_resign/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:58:59 CST 2024] <<<<<< START pulsar cluster in normal mode in owner_resign case >>>>>>' [Mon May 6 16:58:59 CST 2024] <<<<<< START pulsar cluster in normal mode in owner_resign case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/owner_resign/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/owner_resign/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/owner_resign/pulsar-bookie + nc -z localhost 6650 + i=1 + '[' 1 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:59 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bc2ec01b-3f63-4579-89a9-23b411384d97 {"id":"bc2ec01b-3f63-4579-89a9-23b411384d97","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985936} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d2042c384 bc2ec01b-3f63-4579-89a9-23b411384d97 /tidb/cdc/default/default/upstream/7365808406150734168 {"id":7365808406150734168,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bc2ec01b-3f63-4579-89a9-23b411384d97 {"id":"bc2ec01b-3f63-4579-89a9-23b411384d97","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985936} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d2042c384 bc2ec01b-3f63-4579-89a9-23b411384d97 /tidb/cdc/default/default/upstream/7365808406150734168 {"id":7365808406150734168,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bc2ec01b-3f63-4579-89a9-23b411384d97 {"id":"bc2ec01b-3f63-4579-89a9-23b411384d97","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985936} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d2042c384 bc2ec01b-3f63-4579-89a9-23b411384d97 /tidb/cdc/default/default/upstream/7365808406150734168 {"id":7365808406150734168,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x wait process cdc.test exit for 2-th time... + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table ddl_manager.finish_mark not exists for 67-th check, retry later + set +x + tso='449573273741819905 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573273741819905 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:59:00 CST 2024] <<<<<< START cdc server in simple case >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... [Mon May 6 16:59:00 CST 2024] <<<<<< START Pulsar consumer in processor_err_chan case >>>>>> check_changefeed_state http://127.0.0.1:2379 34788aa6-24fd-4e92-92bb-7ca5b5680fc3 normal null public/default ++ date + echo '[Mon May 6 16:59:00 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:59:00 CST 2024] <<<<<< pulsar is ready >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.27972799.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 process cdc.test already exit [Mon May 6 16:59:00 CST 2024] <<<<<< run test case canal_json_content_compatible success! >>>>>> + endpoints=http://127.0.0.1:2379 + changefeed_id=34788aa6-24fd-4e92-92bb-7ca5b5680fc3 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 34788aa6-24fd-4e92-92bb-7ca5b5680fc3 -s + info='{ "upstream_id": 7365808406150734168, "namespace": "default", "id": "34788aa6-24fd-4e92-92bb-7ca5b5680fc3", "state": "normal", "checkpoint_tso": 449573274133463046, "checkpoint_time": "2024-05-06 16:58:59.535", "error": null }' + echo '{ "upstream_id": 7365808406150734168, "namespace": "default", "id": "34788aa6-24fd-4e92-92bb-7ca5b5680fc3", "state": "normal", "checkpoint_tso": 449573274133463046, "checkpoint_time": "2024-05-06 16:58:59.535", "error": null }' { "upstream_id": 7365808406150734168, "namespace": "default", "id": "34788aa6-24fd-4e92-92bb-7ca5b5680fc3", "state": "normal", "checkpoint_tso": 449573274133463046, "checkpoint_time": "2024-05-06 16:58:59.535", "error": null } ++ echo '{' '"upstream_id":' 7365808406150734168, '"namespace":' '"default",' '"id":' '"34788aa6-24fd-4e92-92bb-7ca5b5680fc3",' '"state":' '"normal",' '"checkpoint_tso":' 449573274133463046, '"checkpoint_time":' '"2024-05-06' '16:58:59.535",' '"error":' null '}' ++ jq -r .state + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365808406150734168, '"namespace":' '"default",' '"id":' '"34788aa6-24fd-4e92-92bb-7ca5b5680fc3",' '"state":' '"normal",' '"checkpoint_tso":' 449573274133463046, '"checkpoint_time":' '"2024-05-06' '16:58:59.535",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:00 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/16f81be7-86a1-4cf1-9b2d-49d9987b6c6f {"id":"16f81be7-86a1-4cf1-9b2d-49d9987b6c6f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985937} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d203fe800 16f81be7-86a1-4cf1-9b2d-49d9987b6c6f /tidb/cdc/default/default/upstream/7365808402186459340 {"id":7365808402186459340,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/16f81be7-86a1-4cf1-9b2d-49d9987b6c6f {"id":"16f81be7-86a1-4cf1-9b2d-49d9987b6c6f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985937} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d203fe800 16f81be7-86a1-4cf1-9b2d-49d9987b6c6f /tidb/cdc/default/default/upstream/7365808402186459340 {"id":7365808402186459340,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/16f81be7-86a1-4cf1-9b2d-49d9987b6c6f {"id":"16f81be7-86a1-4cf1-9b2d-49d9987b6c6f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985937} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d203fe800 16f81be7-86a1-4cf1-9b2d-49d9987b6c6f /tidb/cdc/default/default/upstream/7365808402186459340 {"id":7365808402186459340,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x public/default check diff failed 1-th time, retry later ++ date + echo '[Mon May 6 16:59:01 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:59:01 CST 2024] <<<<<< pulsar is ready >>>>>> [Mon May 6 16:59:01 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.33073309.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: a18fd85d-e217-49f0-9fa2-5e26fca64f4f Info: {"upstream_id":7365808414137410017,"namespace":"default","id":"a18fd85d-e217-49f0-9fa2-5e26fca64f4f","sink_uri":"pulsar+ssl://127.0.0.1:6651/ticdc-tiflash-test-14006?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:59:01.156490982+08:00","start_ts":449573271227858945,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"tls-trust-certs-file-path":"/tmp/tidb_cdc_test/tiflash/ca.cert.pem","connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30,"auth-tls-certificate-path":"/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem","auth-tls-private-key-path":"/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem"},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573271227858945,"checkpoint_ts":449573271227858945,"checkpoint_time":"2024-05-06 16:58:48.451"} [Mon May 6 16:59:01 CST 2024] <<<<<< START Pulsar consumer in tiflash case >>>>>> table processor_resolved_ts_fallback.t1 exists [Mon May 6 16:59:01 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.33923394.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:01 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f9720394-e9d1-4060-a504-cb9688b1c0a0 {"id":"f9720394-e9d1-4060-a504-cb9688b1c0a0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985938} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d206388d5 f9720394-e9d1-4060-a504-cb9688b1c0a0 /tidb/cdc/default/default/upstream/7365808443444418038 {"id":7365808443444418038,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f9720394-e9d1-4060-a504-cb9688b1c0a0 {"id":"f9720394-e9d1-4060-a504-cb9688b1c0a0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985938} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d206388d5 f9720394-e9d1-4060-a504-cb9688b1c0a0 /tidb/cdc/default/default/upstream/7365808443444418038 {"id":7365808443444418038,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f9720394-e9d1-4060-a504-cb9688b1c0a0 {"id":"f9720394-e9d1-4060-a504-cb9688b1c0a0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985938} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d206388d5 f9720394-e9d1-4060-a504-cb9688b1c0a0 /tidb/cdc/default/default/upstream/7365808443444418038 {"id":7365808443444418038,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:59:01 CST 2024] <<<<<< START pulsar cluster in ddl_sequence case >>>>>>' [Mon May 6 16:59:01 CST 2024] <<<<<< START pulsar cluster in ddl_sequence case >>>>>> + workdir=/tmp/tidb_cdc_test/ddl_sequence + cluster_type=normal + cd /tmp/tidb_cdc_test/ddl_sequence + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_sequence/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/ddl_sequence/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/ddl_sequence/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_sequence/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/ddl_sequence/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/ddl_sequence/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/ddl_sequence/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/ddl_sequence/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_sequence/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/ddl_sequence/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/ddl_sequence/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/ddl_sequence/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:59:01 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_sequence case >>>>>>' [Mon May 6 16:59:01 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_sequence case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/ddl_sequence/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/ddl_sequence/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/ddl_sequence/pulsar-bookie + nc -z localhost 6650 + i=1 + '[' 1 -gt 20 ']' + sleep 2 [Mon May 6 16:59:01 CST 2024] <<<<<< START Pulsar consumer in gc_safepoint case >>>>>> 0 table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later check diff failed 1-th time, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table ddl_manager.finish_mark not exists for 68-th check, retry later check diff failed 2-th time, retry later + nc -z localhost 6650 + i=3 + '[' 3 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:03 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e50dedd2-0f77-4c24-bebd-00d4a87c29d7 {"id":"e50dedd2-0f77-4c24-bebd-00d4a87c29d7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985940} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d205c29fc e50dedd2-0f77-4c24-bebd-00d4a87c29d7 /tidb/cdc/default/default/upstream/7365808438894456661 {"id":7365808438894456661,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e50dedd2-0f77-4c24-bebd-00d4a87c29d7 {"id":"e50dedd2-0f77-4c24-bebd-00d4a87c29d7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985940} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d205c29fc e50dedd2-0f77-4c24-bebd-00d4a87c29d7 /tidb/cdc/default/default/upstream/7365808438894456661 {"id":7365808438894456661,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e50dedd2-0f77-4c24-bebd-00d4a87c29d7 {"id":"e50dedd2-0f77-4c24-bebd-00d4a87c29d7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985940} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d205c29fc e50dedd2-0f77-4c24-bebd-00d4a87c29d7 /tidb/cdc/default/default/upstream/7365808438894456661 {"id":7365808438894456661,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x ++ date + echo '[Mon May 6 16:59:03 CST 2024] <<<<<< START pulsar cluster in simple case >>>>>>' [Mon May 6 16:59:03 CST 2024] <<<<<< START pulsar cluster in simple case >>>>>> + workdir=/tmp/tidb_cdc_test/simple + cluster_type=normal + cd /tmp/tidb_cdc_test/simple + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/simple/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/simple/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/simple/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/simple/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/simple/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/simple/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/simple/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/simple/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/simple/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/simple/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/simple/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/simple/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:59:03 CST 2024] <<<<<< START pulsar cluster in normal mode in simple case >>>>>>' [Mon May 6 16:59:03 CST 2024] <<<<<< START pulsar cluster in normal mode in simple case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/simple/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/simple/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/simple/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 Tenant not found Reason: Tenant not found table cdc_tiflash_test.multi_data_type exists check diff failed 1-th time, retry later + i=1 + '[' 1 -gt 20 ']' + sleep 2 public/default ++ date + echo '[Mon May 6 16:59:03 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:59:03 CST 2024] <<<<<< pulsar is ready >>>>>> check diff failed 2-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.3354.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:04 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/214ca646-afdf-4155-af0a-105892089956 {"id":"214ca646-afdf-4155-af0a-105892089956","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985941} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d204e1104 214ca646-afdf-4155-af0a-105892089956 /tidb/cdc/default/default/upstream/7365808414695768478 {"id":7365808414695768478,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/214ca646-afdf-4155-af0a-105892089956 {"id":"214ca646-afdf-4155-af0a-105892089956","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985941} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d204e1104 214ca646-afdf-4155-af0a-105892089956 /tidb/cdc/default/default/upstream/7365808414695768478 {"id":7365808414695768478,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/214ca646-afdf-4155-af0a-105892089956 {"id":"214ca646-afdf-4155-af0a-105892089956","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985941} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d204e1104 214ca646-afdf-4155-af0a-105892089956 /tidb/cdc/default/default/upstream/7365808414695768478 {"id":7365808414695768478,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 16:59:05 CST 2024] <<<<<< START Pulsar consumer in changefeed_finish case >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:04 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/6760f65e-620c-469e-9d8a-d332106a54ce {UpstreamID:7365808405637481761 Namespace:default ID:6760f65e-620c-469e-9d8a-d332106a54ce SinkURI:pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-15533?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 16:58:56.421598386 +0800 CST StartTs:449573273142820868 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00389c360 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573273313214467} {CheckpointTs:449573275226865674 MinTableBarrierTs:449573275226865674 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a8c5703e-6860-472a-b609-922c6f8e0093 {"id":"a8c5703e-6860-472a-b609-922c6f8e0093","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985941} /tidb/cdc/default/__cdc_meta__/capture/cc0de295-8c77-4c9d-8d6f-d7696ad0a76b {"id":"cc0de295-8c77-4c9d-8d6f-d7696ad0a76b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985926} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d203f21cb cc0de295-8c77-4c9d-8d6f-d7696ad0a76b /tidb/cdc/default/__cdc_meta__/owner/22318f4d203f2279 a8c5703e-6860-472a-b609-922c6f8e0093 /tidb/cdc/default/default/changefeed/info/6760f65e-620c-469e-9d8a-d332106a54ce {"upstream-id":7365808405637481761,"namespace":"default","changefeed-id":"6760f65e-620c-469e-9d8a-d332106a54ce","sink-uri":"pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-15533?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T16:58:56.421598386+08:00","start-ts":449573273142820868,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573273313214467} /tidb/cdc/default/default/changefeed/status/6760f65e-620c-469e-9d8a-d332106a54ce {"checkpoint-ts":449573275226865674,"min-table-barrier-ts":449573275226865674,"admin-job-type":0} /tidb/cdc/default/default/task/position/a8c5703e-6860-472a-b609-922c6f8e0093/6760f65e-620c-469e-9d8a-d332106a54ce {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cc0de295-8c77-4c9d-8d6f-d7696ad0a76b/6760f65e-620c-469e-9d8a-d332106a54ce {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808405637481761 {"id":7365808405637481761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/6760f65e-620c-469e-9d8a-d332106a54ce {UpstreamID:7365808405637481761 Namespace:default ID:6760f65e-620c-469e-9d8a-d332106a54ce SinkURI:pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-15533?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 16:58:56.421598386 +0800 CST StartTs:449573273142820868 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00389c360 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573273313214467} {CheckpointTs:449573275226865674 MinTableBarrierTs:449573275226865674 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a8c5703e-6860-472a-b609-922c6f8e0093 {"id":"a8c5703e-6860-472a-b609-922c6f8e0093","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985941} /tidb/cdc/default/__cdc_meta__/capture/cc0de295-8c77-4c9d-8d6f-d7696ad0a76b {"id":"cc0de295-8c77-4c9d-8d6f-d7696ad0a76b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985926} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d203f21cb cc0de295-8c77-4c9d-8d6f-d7696ad0a76b /tidb/cdc/default/__cdc_meta__/owner/22318f4d203f2279 a8c5703e-6860-472a-b609-922c6f8e0093 /tidb/cdc/default/default/changefeed/info/6760f65e-620c-469e-9d8a-d332106a54ce {"upstream-id":7365808405637481761,"namespace":"default","changefeed-id":"6760f65e-620c-469e-9d8a-d332106a54ce","sink-uri":"pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-15533?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T16:58:56.421598386+08:00","start-ts":449573273142820868,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573273313214467} /tidb/cdc/default/default/changefeed/status/6760f65e-620c-469e-9d8a-d332106a54ce {"checkpoint-ts":449573275226865674,"min-table-barrier-ts":449573275226865674,"admin-job-type":0} /tidb/cdc/default/default/task/position/a8c5703e-6860-472a-b609-922c6f8e0093/6760f65e-620c-469e-9d8a-d332106a54ce {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cc0de295-8c77-4c9d-8d6f-d7696ad0a76b/6760f65e-620c-469e-9d8a-d332106a54ce {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808405637481761 {"id":7365808405637481761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/6760f65e-620c-469e-9d8a-d332106a54ce {UpstreamID:7365808405637481761 Namespace:default ID:6760f65e-620c-469e-9d8a-d332106a54ce SinkURI:pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-15533?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 16:58:56.421598386 +0800 CST StartTs:449573273142820868 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00389c360 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573273313214467} {CheckpointTs:449573275226865674 MinTableBarrierTs:449573275226865674 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a8c5703e-6860-472a-b609-922c6f8e0093 {"id":"a8c5703e-6860-472a-b609-922c6f8e0093","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985941} /tidb/cdc/default/__cdc_meta__/capture/cc0de295-8c77-4c9d-8d6f-d7696ad0a76b {"id":"cc0de295-8c77-4c9d-8d6f-d7696ad0a76b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985926} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d203f21cb cc0de295-8c77-4c9d-8d6f-d7696ad0a76b /tidb/cdc/default/__cdc_meta__/owner/22318f4d203f2279 a8c5703e-6860-472a-b609-922c6f8e0093 /tidb/cdc/default/default/changefeed/info/6760f65e-620c-469e-9d8a-d332106a54ce {"upstream-id":7365808405637481761,"namespace":"default","changefeed-id":"6760f65e-620c-469e-9d8a-d332106a54ce","sink-uri":"pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-15533?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T16:58:56.421598386+08:00","start-ts":449573273142820868,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573273313214467} /tidb/cdc/default/default/changefeed/status/6760f65e-620c-469e-9d8a-d332106a54ce {"checkpoint-ts":449573275226865674,"min-table-barrier-ts":449573275226865674,"admin-job-type":0} /tidb/cdc/default/default/task/position/a8c5703e-6860-472a-b609-922c6f8e0093/6760f65e-620c-469e-9d8a-d332106a54ce {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cc0de295-8c77-4c9d-8d6f-d7696ad0a76b/6760f65e-620c-469e-9d8a-d332106a54ce {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808405637481761 {"id":7365808405637481761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table processor_resolved_ts_fallback.t2 not exists for 1-th check, retry later check diff failed 3-th time, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table ddl_manager.finish_mark not exists for 69-th check, retry later + nc -z localhost 6650 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 check diff failed 1-th time, retry later + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public check diff successfully check diff failed 3-th time, retry later + /usr/local/pulsar/bin/pulsar-admin namespaces list public wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 70-th check, retry later wait process cdc.test exit for 2-th time... + set +x + tso='449573275686666241 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573275686666241 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:59:07 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.34003402.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_resolved_ts_fallback.t2 exists table processor_resolved_ts_fallback.t3 not exists for 1-th check, retry later + nc -z localhost 6650 + i=3 + '[' 3 -gt 20 ']' + sleep 2 wait process cdc.test exit for 3-th time... check diff successfully wait process cdc.test exit for 1-th time... check diff successfully check_safepoint_forward http://127.0.0.1:2379 7365808402186459340 449573276078571525 449573274387480581 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:59:07 CST 2024] <<<<<< run test case tiflash success! >>>>>> check diff failed 2-th time, retry later wait process cdc.test exit for 2-th time... Tenant not found Reason: Tenant not found public/default ++ date + echo '[Mon May 6 16:59:08 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:59:08 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.3427.out cli changefeed create --start-ts=449573272278532097 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-resolve-lock-test-32395?protocol=canal-json&enable-tidb-extension=true' run task successfully + i=1 + '[' 1 -gt 20 ']' + sleep 2 wait process cdc.test exit for 3-th time... table ddl_manager.finish_mark not exists for 71-th check, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public public/default ++ date + echo '[Mon May 6 16:59:08 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:59:08 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.3401.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-batch-add-table-test-27399?protocol=canal-json&enable-tidb-extension=true' check_changefeed_state http://127.0.0.1:2379 6136261c-db45-4fc2-9ba3-950144d0e910 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=6136261c-db45-4fc2-9ba3-950144d0e910 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 6136261c-db45-4fc2-9ba3-950144d0e910 -s + info='{ "upstream_id": 7365808402186459340, "namespace": "default", "id": "6136261c-db45-4fc2-9ba3-950144d0e910", "state": "stopped", "checkpoint_tso": 449573276340453383, "checkpoint_time": "2024-05-06 16:59:07.954", "error": null }' + echo '{ "upstream_id": 7365808402186459340, "namespace": "default", "id": "6136261c-db45-4fc2-9ba3-950144d0e910", "state": "stopped", "checkpoint_tso": 449573276340453383, "checkpoint_time": "2024-05-06 16:59:07.954", "error": null }' { "upstream_id": 7365808402186459340, "namespace": "default", "id": "6136261c-db45-4fc2-9ba3-950144d0e910", "state": "stopped", "checkpoint_tso": 449573276340453383, "checkpoint_time": "2024-05-06 16:59:07.954", "error": null } ++ echo '{' '"upstream_id":' 7365808402186459340, '"namespace":' '"default",' '"id":' '"6136261c-db45-4fc2-9ba3-950144d0e910",' '"state":' '"stopped",' '"checkpoint_tso":' 449573276340453383, '"checkpoint_time":' '"2024-05-06' '16:59:07.954",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365808402186459340, '"namespace":' '"default",' '"id":' '"6136261c-db45-4fc2-9ba3-950144d0e910",' '"state":' '"stopped",' '"checkpoint_tso":' 449573276340453383, '"checkpoint_time":' '"2024-05-06' '16:59:07.954",' '"error":' null '}' ++ jq -r .error.message cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:59:09 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> Create changefeed successfully! ID: e670f99b-7523-4faf-8eb2-c22149b0ada0 Info: {"upstream_id":7365808427702158470,"namespace":"default","id":"e670f99b-7523-4faf-8eb2-c22149b0ada0","sink_uri":"pulsar://127.0.0.1:6650/ticdc-resolve-lock-test-32395?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:59:09.699216048+08:00","start_ts":449573272278532097,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573272278532097,"checkpoint_ts":449573272278532097,"checkpoint_time":"2024-05-06 16:58:52.459"} PASS Create changefeed successfully! ID: 433fcd8f-4452-4ced-a289-89315451e621 Info: {"upstream_id":7365808441496147259,"namespace":"default","id":"433fcd8f-4452-4ced-a289-89315451e621","sink_uri":"pulsar://127.0.0.1:6650/ticdc-batch-add-table-test-27399?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:59:09.649821695+08:00","start_ts":449573276602859525,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573276602859525,"checkpoint_ts":449573276602859525,"checkpoint_time":"2024-05-06 16:59:08.955"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365808402186459340 table processor_resolved_ts_fallback.t3 exists cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^2$' 2 run task successfully killing cdc1: 2768 cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 1-th time, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:10 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd69105c-92cf-499d-9cff-db953f9973f7 {"id":"bd69105c-92cf-499d-9cff-db953f9973f7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985947} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d2057c426 bd69105c-92cf-499d-9cff-db953f9973f7 /tidb/cdc/default/default/upstream/7365808433318085220 {"id":7365808433318085220,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd69105c-92cf-499d-9cff-db953f9973f7 {"id":"bd69105c-92cf-499d-9cff-db953f9973f7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985947} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d2057c426 bd69105c-92cf-499d-9cff-db953f9973f7 /tidb/cdc/default/default/upstream/7365808433318085220 {"id":7365808433318085220,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd69105c-92cf-499d-9cff-db953f9973f7 {"id":"bd69105c-92cf-499d-9cff-db953f9973f7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985947} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d2057c426 bd69105c-92cf-499d-9cff-db953f9973f7 /tidb/cdc/default/default/upstream/7365808433318085220 {"id":7365808433318085220,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Tenant not found Reason: Tenant not found + i=1 + '[' 1 -gt 20 ']' + sleep 2 + set +x [Mon May 6 16:59:11 CST 2024] <<<<<< START Pulsar consumer in batch_add_table case >>>>>> table ddl_manager.finish_mark not exists for 72-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + set +x [Mon May 6 16:59:11 CST 2024] <<<<<< START Pulsar consumer in resolve_lock case >>>>>> go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b table batch_add_table.finish_mark not exists for 1-th check, retry later + /usr/local/pulsar/bin/pulsar-admin namespaces list public go: downloading go.uber.org/multierr v1.11.0 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/docker/go-units v0.5.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading golang.org/x/net v0.24.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda [Mon May 6 16:59:11 CST 2024] <<<<<< START Pulsar consumer in capture_session_done_during_task case >>>>>> Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 2-th time, retry later go: downloading golang.org/x/text v0.14.0 lease 22318f4d2057c426 revoked + /usr/local/pulsar/bin/pulsar-admin namespaces list public check diff successfully run task successfully check_changefeed_state http://127.0.0.1:2379 6136261c-db45-4fc2-9ba3-950144d0e910 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=6136261c-db45-4fc2-9ba3-950144d0e910 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 6136261c-db45-4fc2-9ba3-950144d0e910 -s table ddl_manager.finish_mark not exists for 73-th check, retry later table capture_session_done_during_task.t exists check diff failed 1-th time, retry later table batch_add_table.finish_mark not exists for 2-th check, retry later + info='{ "upstream_id": 7365808402186459340, "namespace": "default", "id": "6136261c-db45-4fc2-9ba3-950144d0e910", "state": "normal", "checkpoint_tso": 449573276340453383, "checkpoint_time": "2024-05-06 16:59:07.954", "error": null }' + echo '{ "upstream_id": 7365808402186459340, "namespace": "default", "id": "6136261c-db45-4fc2-9ba3-950144d0e910", "state": "normal", "checkpoint_tso": 449573276340453383, "checkpoint_time": "2024-05-06 16:59:07.954", "error": null }' { "upstream_id": 7365808402186459340, "namespace": "default", "id": "6136261c-db45-4fc2-9ba3-950144d0e910", "state": "normal", "checkpoint_tso": 449573276340453383, "checkpoint_time": "2024-05-06 16:59:07.954", "error": null } ++ echo '{' '"upstream_id":' 7365808402186459340, '"namespace":' '"default",' '"id":' '"6136261c-db45-4fc2-9ba3-950144d0e910",' '"state":' '"normal",' '"checkpoint_tso":' 449573276340453383, '"checkpoint_time":' '"2024-05-06' '16:59:07.954",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365808402186459340, '"namespace":' '"default",' '"id":' '"6136261c-db45-4fc2-9ba3-950144d0e910",' '"state":' '"normal",' '"checkpoint_tso":' 449573276340453383, '"checkpoint_time":' '"2024-05-06' '16:59:07.954",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7365808402186459340 449573276340453382 449573276340453383 public/default ++ date + echo '[Mon May 6 16:59:13 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:59:13 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.11331.out cli changefeed create --start-ts=449573273318719489 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-ddl-sequence-test-22868?protocol=canal-json&enable-tidb-extension=true' Tenant not found Reason: Tenant not found go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading golang.org/x/tools v0.20.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/fatih/color v1.16.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c + i=1 + '[' 1 -gt 20 ']' + sleep 2 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... run task successfully check_changefeed_state http://127.0.0.1:2379 6136261c-db45-4fc2-9ba3-950144d0e910 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=6136261c-db45-4fc2-9ba3-950144d0e910 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 6136261c-db45-4fc2-9ba3-950144d0e910 -s go: downloading k8s.io/apimachinery v0.28.6 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b + info='{ "upstream_id": 7365808402186459340, "namespace": "default", "id": "6136261c-db45-4fc2-9ba3-950144d0e910", "state": "stopped", "checkpoint_tso": 449573276340453383, "checkpoint_time": "2024-05-06 16:59:07.954", "error": null }' + echo '{ "upstream_id": 7365808402186459340, "namespace": "default", "id": "6136261c-db45-4fc2-9ba3-950144d0e910", "state": "stopped", "checkpoint_tso": 449573276340453383, "checkpoint_time": "2024-05-06 16:59:07.954", "error": null }' { "upstream_id": 7365808402186459340, "namespace": "default", "id": "6136261c-db45-4fc2-9ba3-950144d0e910", "state": "stopped", "checkpoint_tso": 449573276340453383, "checkpoint_time": "2024-05-06 16:59:07.954", "error": null } ++ echo '{' '"upstream_id":' 7365808402186459340, '"namespace":' '"default",' '"id":' '"6136261c-db45-4fc2-9ba3-950144d0e910",' '"state":' '"stopped",' '"checkpoint_tso":' 449573276340453383, '"checkpoint_time":' '"2024-05-06' '16:59:07.954",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365808402186459340, '"namespace":' '"default",' '"id":' '"6136261c-db45-4fc2-9ba3-950144d0e910",' '"state":' '"stopped",' '"checkpoint_tso":' 449573276340453383, '"checkpoint_time":' '"2024-05-06' '16:59:07.954",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully Create changefeed successfully! ID: 7f6a394d-b2ef-4f87-8f56-a3b02193c64e Info: {"upstream_id":7365808443444418038,"namespace":"default","id":"7f6a394d-b2ef-4f87-8f56-a3b02193c64e","sink_uri":"pulsar://127.0.0.1:6650/ticdc-ddl-sequence-test-22868?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:59:14.594087273+08:00","start_ts":449573273318719489,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573273318719489,"checkpoint_ts":449573273318719489,"checkpoint_time":"2024-05-06 16:58:56.427"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 74-th check, retry later check_changefeed_state http://127.0.0.1:2379 27878695-cc1a-4df2-b1ea-35f542f3d613 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=27878695-cc1a-4df2-b1ea-35f542f3d613 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 27878695-cc1a-4df2-b1ea-35f542f3d613 -s + info='{ "upstream_id": 7365808402186459340, "namespace": "default", "id": "27878695-cc1a-4df2-b1ea-35f542f3d613", "state": "normal", "checkpoint_tso": 449573278188568583, "checkpoint_time": "2024-05-06 16:59:15.004", "error": null }' + echo '{ "upstream_id": 7365808402186459340, "namespace": "default", "id": "27878695-cc1a-4df2-b1ea-35f542f3d613", "state": "normal", "checkpoint_tso": 449573278188568583, "checkpoint_time": "2024-05-06 16:59:15.004", "error": null }' { "upstream_id": 7365808402186459340, "namespace": "default", "id": "27878695-cc1a-4df2-b1ea-35f542f3d613", "state": "normal", "checkpoint_tso": 449573278188568583, "checkpoint_time": "2024-05-06 16:59:15.004", "error": null } ++ echo '{' '"upstream_id":' 7365808402186459340, '"namespace":' '"default",' '"id":' '"27878695-cc1a-4df2-b1ea-35f542f3d613",' '"state":' '"normal",' '"checkpoint_tso":' 449573278188568583, '"checkpoint_time":' '"2024-05-06' '16:59:15.004",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365808402186459340, '"namespace":' '"default",' '"id":' '"27878695-cc1a-4df2-b1ea-35f542f3d613",' '"state":' '"normal",' '"checkpoint_tso":' 449573278188568583, '"checkpoint_time":' '"2024-05-06' '16:59:15.004",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365808402186459340 check diff failed 2-th time, retry later table batch_add_table.finish_mark exists check diff successfully public/default wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_topics/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:59:13 CST 2024] <<<<<< run test case multi_topics success! >>>>>> cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 3-th time, retry later + /usr/local/pulsar/bin/pulsar-admin namespaces list public ++ date + echo '[Mon May 6 16:59:16 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:59:16 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.cli.3257.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-owner-resign-test-4235?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 + set +x [Mon May 6 16:59:16 CST 2024] <<<<<< START Pulsar consumer in ddl_sequence case >>>>>> wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 75-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 16:59:16 CST 2024] <<<<<< run test case batch_add_table success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/avro_basic/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:59:17 CST 2024] <<<<<< run test case avro_basic success! >>>>>> check diff failed 3-th time, retry later Create changefeed successfully! ID: 1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 Info: {"upstream_id":7365808414652405965,"namespace":"default","id":"1daf36e8-aca9-4565-aa7e-2f11d7cd11d9","sink_uri":"pulsar://127.0.0.1:6650/ticdc-owner-resign-test-4235?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:59:17.721300225+08:00","start_ts":449573278619533320,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573278619533320,"checkpoint_ts":449573278619533320,"checkpoint_time":"2024-05-06 16:59:16.648"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 run task successfully table ddl_manager.finish_mark not exists for 76-th check, retry later Changefeed remove successfully. ID: 6136261c-db45-4fc2-9ba3-950144d0e910 CheckpointTs: 449573276340453383 SinkURI: pulsar://127.0.0.1:6650/ticdc-gc-safepoint-18063?protocol=canal-json&enable-tidb-extension=true check_safepoint_forward http://127.0.0.1:2379 7365808402186459340 449573276340453382 449573278188568583 449573276340453383 check diff successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... check diff failed 1-th time, retry later + set +x [Mon May 6 16:59:19 CST 2024] <<<<<< START Pulsar consumer in owner_resign case >>>>>> cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... 1 run task successfully table owner_resign.t1 not exists for 1-th check, retry later run task successfully public/default ++ date + echo '[Mon May 6 16:59:20 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:59:20 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.3331.out cli changefeed create --start-ts=449573273741819905 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-simple-test-24447?protocol=canal-json&enable-tidb-extension=true' Changefeed remove successfully. ID: 27878695-cc1a-4df2-b1ea-35f542f3d613 CheckpointTs: 449573279237406722 SinkURI: pulsar://127.0.0.1:6650/ticdc-gc-safepoint-18063?protocol=canal-json&enable-tidb-extension=true check_safepoint_cleared http://127.0.0.1:2379 7365808402186459340 run task successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 77-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... Create changefeed successfully! ID: 4117735f-4e8b-41fd-83b6-5e882115f5a1 Info: {"upstream_id":7365808438894456661,"namespace":"default","id":"4117735f-4e8b-41fd-83b6-5e882115f5a1","sink_uri":"pulsar://127.0.0.1:6650/ticdc-simple-test-24447?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:59:21.7317823+08:00","start_ts":449573273741819905,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573273741819905,"checkpoint_ts":449573273741819905,"checkpoint_time":"2024-05-06 16:58:58.041"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table owner_resign.t1 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 3-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' 1 run task successfully check diff failed 1-th time, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:59:22 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 78-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 16:59:23 CST 2024] <<<<<< run test case capture_session_done_during_task success! >>>>>> + set +x [Mon May 6 16:59:23 CST 2024] <<<<<< START Pulsar consumer in simple case >>>>>> succeed to verify meta placement rules table ddl_sequence.finish_mark not exists for 1-th check, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... table ddl_sequence.finish_mark not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully [Mon May 6 16:59:24 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerDontUpdateCheckpoint=return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.34203422.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 79-th check, retry later wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 16:59:25 CST 2024] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 80-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:59:26 CST 2024] <<<<<< run test case canal_json_handle_key_only success! >>>>>> table ddl_sequence.finish_mark not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:27 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 {UpstreamID:7365808414652405965 Namespace:default ID:1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 SinkURI:pulsar://127.0.0.1:6650/ticdc-owner-resign-test-4235?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 16:59:17.721300225 +0800 CST StartTs:449573278619533320 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00305e480 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573278894784517} {CheckpointTs:449573281214758924 MinTableBarrierTs:449573281490010113 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b9ea401-a5ba-46ec-95a6-48d40631e240 {"id":"8b9ea401-a5ba-46ec-95a6-48d40631e240","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985936} /tidb/cdc/default/__cdc_meta__/capture/9acbf17f-7e29-4c12-99c5-db0bd6787bcd {"id":"9acbf17f-7e29-4c12-99c5-db0bd6787bcd","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985965} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d204c92e3 8b9ea401-a5ba-46ec-95a6-48d40631e240 /tidb/cdc/default/__cdc_meta__/owner/22318f4d204c93fa 9acbf17f-7e29-4c12-99c5-db0bd6787bcd /tidb/cdc/default/default/changefeed/info/1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 {"upstream-id":7365808414652405965,"namespace":"default","changefeed-id":"1daf36e8-aca9-4565-aa7e-2f11d7cd11d9","sink-uri":"pulsar://127.0.0.1:6650/ticdc-owner-resign-test-4235?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T16:59:17.721300225+08:00","start-ts":449573278619533320,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573278894784517} /tidb/cdc/default/default/changefeed/status/1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 {"checkpoint-ts":449573281214758924,"min-table-barrier-ts":449573281490010113,"admin-job-type":0} /tidb/cdc/default/default/task/position/8b9ea401-a5ba-46ec-95a6-48d40631e240/1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/9acbf17f-7e29-4c12-99c5-db0bd6787bcd/1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808414652405965 {"id":7365808414652405965,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 {UpstreamID:7365808414652405965 Namespace:default ID:1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 SinkURI:pulsar://127.0.0.1:6650/ticdc-owner-resign-test-4235?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 16:59:17.721300225 +0800 CST StartTs:449573278619533320 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00305e480 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573278894784517} {CheckpointTs:449573281214758924 MinTableBarrierTs:449573281490010113 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b9ea401-a5ba-46ec-95a6-48d40631e240 {"id":"8b9ea401-a5ba-46ec-95a6-48d40631e240","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985936} /tidb/cdc/default/__cdc_meta__/capture/9acbf17f-7e29-4c12-99c5-db0bd6787bcd {"id":"9acbf17f-7e29-4c12-99c5-db0bd6787bcd","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985965} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d204c92e3 8b9ea401-a5ba-46ec-95a6-48d40631e240 /tidb/cdc/default/__cdc_meta__/owner/22318f4d204c93fa 9acbf17f-7e29-4c12-99c5-db0bd6787bcd /tidb/cdc/default/default/changefeed/info/1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 {"upstream-id":7365808414652405965,"namespace":"default","changefeed-id":"1daf36e8-aca9-4565-aa7e-2f11d7cd11d9","sink-uri":"pulsar://127.0.0.1:6650/ticdc-owner-resign-test-4235?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T16:59:17.721300225+08:00","start-ts":449573278619533320,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573278894784517} /tidb/cdc/default/default/changefeed/status/1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 {"checkpoint-ts":449573281214758924,"min-table-barrier-ts":449573281490010113,"admin-job-type":0} /tidb/cdc/default/default/task/position/8b9ea401-a5ba-46ec-95a6-48d40631e240/1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/9acbf17f-7e29-4c12-99c5-db0bd6787bcd/1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808414652405965 {"id":7365808414652405965,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 {UpstreamID:7365808414652405965 Namespace:default ID:1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 SinkURI:pulsar://127.0.0.1:6650/ticdc-owner-resign-test-4235?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 16:59:17.721300225 +0800 CST StartTs:449573278619533320 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00305e480 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573278894784517} {CheckpointTs:449573281214758924 MinTableBarrierTs:449573281490010113 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b9ea401-a5ba-46ec-95a6-48d40631e240 {"id":"8b9ea401-a5ba-46ec-95a6-48d40631e240","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985936} /tidb/cdc/default/__cdc_meta__/capture/9acbf17f-7e29-4c12-99c5-db0bd6787bcd {"id":"9acbf17f-7e29-4c12-99c5-db0bd6787bcd","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985965} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d204c92e3 8b9ea401-a5ba-46ec-95a6-48d40631e240 /tidb/cdc/default/__cdc_meta__/owner/22318f4d204c93fa 9acbf17f-7e29-4c12-99c5-db0bd6787bcd /tidb/cdc/default/default/changefeed/info/1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 {"upstream-id":7365808414652405965,"namespace":"default","changefeed-id":"1daf36e8-aca9-4565-aa7e-2f11d7cd11d9","sink-uri":"pulsar://127.0.0.1:6650/ticdc-owner-resign-test-4235?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T16:59:17.721300225+08:00","start-ts":449573278619533320,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573278894784517} /tidb/cdc/default/default/changefeed/status/1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 {"checkpoint-ts":449573281214758924,"min-table-barrier-ts":449573281490010113,"admin-job-type":0} /tidb/cdc/default/default/task/position/8b9ea401-a5ba-46ec-95a6-48d40631e240/1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/9acbf17f-7e29-4c12-99c5-db0bd6787bcd/1daf36e8-aca9-4565-aa7e-2f11d7cd11d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808414652405965 {"id":7365808414652405965,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^2$' 2 run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348480fc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq, pid:10800, start at 2024-05-06 16:59:27.70800313 +0800 CST m=+5.366707907 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:27.714 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:27.679 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:27.679 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 81-th check, retry later check data successfully wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_rocks/run.sh using Sink-Type: pulsar... <<================= table ddl_sequence.finish_mark not exists for 4-th check, retry later wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:59:29 CST 2024] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348480fc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq, pid:10800, start at 2024-05-06 16:59:27.70800313 +0800 CST m=+5.366707907 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:27.714 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:27.679 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:27.679 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348482880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-0488-qjskq, pid:10878, start at 2024-05-06 16:59:27.822248492 +0800 CST m=+5.419453909 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:27.830 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:27.828 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:27.828 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:59:30 CST 2024] <<<<<< run test case simple success! >>>>>> [2024/05/06 16:59:29.824 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc001dfc780/127.0.0.1:2379] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"] Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } \033[0;36m<<< Run all test success >>>\033[0m table ddl_sequence.finish_mark exists check diff successfully table ddl_manager.finish_mark not exists for 82-th check, retry later [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } check diff failed 1-th time, retry later [Pipeline] // timeout [Pipeline] } wait process cdc.test exit for 1-th time... [Pipeline] // stage [Pipeline] } [Pipeline] // container ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.12272.out cli tso query --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:59:33 CST 2024] <<<<<< run test case ddl_sequence success! >>>>>> table ddl_manager.finish_mark not exists for 83-th check, retry later check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3484d5440018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-x5q3-r7q9p, pid:4529, start at 2024-05-06 16:59:33.095940204 +0800 CST m=+5.598176415 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:33.104 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:33.073 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:33.073 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449573282913714178 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573282913714178 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:59:34 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.1231612318.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark exists check diff successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/canal_json_claim_check/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:59:34 CST 2024] <<<<<< run test case canal_json_claim_check success! >>>>>> wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3484d5440018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-x5q3-r7q9p, pid:4529, start at 2024-05-06 16:59:33.095940204 +0800 CST m=+5.598176415 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:33.104 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:33.073 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:33.073 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3484d6b00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-x5q3-r7q9p, pid:4601, start at 2024-05-06 16:59:33.183382871 +0800 CST m=+5.630508355 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:33.191 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:33.164 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:33.164 +0800 All versions after safe point can be accessed. (DO NOT EDIT) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc already exists) [Pipeline] // cache Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] } check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 [Pipeline] // dir [Pipeline] } [Pipeline] // timeout wait process cdc.test exit for 2-th time... [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3484dcdc0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-j6m9-tmj32, pid:4886, start at 2024-05-06 16:59:33.572290522 +0800 CST m=+5.518317239 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:33.582 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:33.559 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:33.559 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3484dcdc0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-j6m9-tmj32, pid:4886, start at 2024-05-06 16:59:33.572290522 +0800 CST m=+5.518317239 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:33.582 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:33.559 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:33.559 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3484dd9c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-j6m9-tmj32, pid:4968, start at 2024-05-06 16:59:33.651857862 +0800 CST m=+5.535171214 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:33.660 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:33.658 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:33.658 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // withEnv [Pipeline] } wait process cdc.test exit for 3-th time... [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:59:37 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:37 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/164a76bb-9e61-4606-8d08-4d55d29bed4c {"id":"164a76bb-9e61-4606-8d08-4d55d29bed4c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985975} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eca7d7 164a76bb-9e61-4606-8d08-4d55d29bed4c /tidb/cdc/default/default/upstream/7365808601977377040 {"id":7365808601977377040,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/164a76bb-9e61-4606-8d08-4d55d29bed4c {"id":"164a76bb-9e61-4606-8d08-4d55d29bed4c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985975} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eca7d7 164a76bb-9e61-4606-8d08-4d55d29bed4c /tidb/cdc/default/default/upstream/7365808601977377040 {"id":7365808601977377040,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/164a76bb-9e61-4606-8d08-4d55d29bed4c {"id":"164a76bb-9e61-4606-8d08-4d55d29bed4c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985975} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eca7d7 164a76bb-9e61-4606-8d08-4d55d29bed4c /tidb/cdc/default/default/upstream/7365808601977377040 {"id":7365808601977377040,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:59:37 CST 2024] <<<<<< START pulsar cluster in partition_table case >>>>>>' [Mon May 6 16:59:37 CST 2024] <<<<<< START pulsar cluster in partition_table case >>>>>> + workdir=/tmp/tidb_cdc_test/partition_table + cluster_type=normal + cd /tmp/tidb_cdc_test/partition_table + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/partition_table/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/partition_table/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/partition_table/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/partition_table/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/partition_table/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/partition_table/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/partition_table/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/partition_table/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/partition_table/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/partition_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/partition_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/partition_table/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:59:37 CST 2024] <<<<<< START pulsar cluster in normal mode in partition_table case >>>>>>' [Mon May 6 16:59:37 CST 2024] <<<<<< START pulsar cluster in normal mode in partition_table case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/partition_table/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/partition_table/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/partition_table/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 [Mon May 6 16:59:37 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.59915993.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 16:59:32.285 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc001dfc780/127.0.0.1:2379] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"] [2024/05/06 16:59:32.504 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc001b021e0/127.0.0.1:2479] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"] [2024/05/06 16:59:34.834 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc001dfc780/127.0.0.1:2379] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"] [2024/05/06 16:59:35.081 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc001b021e0/127.0.0.1:2479] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"] [2024/05/06 16:59:37.919 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc001b021e0/127.0.0.1:2479] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"] [2024/05/06 16:59:38.446 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc001dfc780/127.0.0.1:2379] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"] [2024/05/06 16:59:38.499 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc001a64000/127.0.0.1:2379] [attempt=0] [error="rpc error: code = NotFound desc = etcdserver: requested lease not found"] [2024/05/06 16:59:38.500 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc001f62000/127.0.0.1:2479] [attempt=0] [error="rpc error: code = NotFound desc = etcdserver: requested lease not found"] {"level":"warn","ts":1714985978.5016587,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0046c5c00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = NotFound desc = etcdserver: requested lease not found"} ++ date + echo '[Mon May 6 16:59:38 CST 2024] <<<<<< START pulsar cluster in changefeed_reconstruct case >>>>>>' [Mon May 6 16:59:38 CST 2024] <<<<<< START pulsar cluster in changefeed_reconstruct case >>>>>> + workdir=/tmp/tidb_cdc_test/changefeed_reconstruct + cluster_type=normal + cd /tmp/tidb_cdc_test/changefeed_reconstruct + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/changefeed_reconstruct/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/changefeed_reconstruct/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/changefeed_reconstruct/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/changefeed_reconstruct/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/changefeed_reconstruct/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/changefeed_reconstruct/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/changefeed_reconstruct/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:59:38 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_reconstruct case >>>>>>' [Mon May 6 16:59:38 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_reconstruct case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/changefeed_reconstruct/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/changefeed_reconstruct/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/changefeed_reconstruct/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:40 GMT < Content-Length: 860 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f083c0bf-65b8-40d0-a908-d82c1f39eae9 {"id":"f083c0bf-65b8-40d0-a908-d82c1f39eae9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985978} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d210b8031 f083c0bf-65b8-40d0-a908-d82c1f39eae9 /tidb/cdc/default/default/upstream/7365808629334094005 {"id":7365808629334094005,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f083c0bf-65b8-40d0-a908-d82c1f39eae9 {"id":"f083c0bf-65b8-40d0-a908-d82c1f39eae9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985978} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d210b8031 f083c0bf-65b8-40d0-a908-d82c1f39eae9 /tidb/cdc/default/default/upstream/7365808629334094005 {"id":7365808629334094005,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f083c0bf-65b8-40d0-a908-d82c1f39eae9 {"id":"f083c0bf-65b8-40d0-a908-d82c1f39eae9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985978} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d210b8031 f083c0bf-65b8-40d0-a908-d82c1f39eae9 /tidb/cdc/default/default/upstream/7365808629334094005 {"id":7365808629334094005,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:59:40 CST 2024] <<<<<< START pulsar cluster in region_merge case >>>>>>' [Mon May 6 16:59:40 CST 2024] <<<<<< START pulsar cluster in region_merge case >>>>>> + workdir=/tmp/tidb_cdc_test/region_merge + cluster_type=mtls + cd /tmp/tidb_cdc_test/region_merge + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/region_merge/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/region_merge/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/region_merge/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/region_merge/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/region_merge/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/region_merge/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/region_merge/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/region_merge/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/region_merge/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/region_merge/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/region_merge/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/region_merge/pulsar_standalone.conf + pulsar_port=6650 + '[' mtls == mtls ']' + pulsar_port=6651 + gen_mtls_config + openssl genrsa -out ca.key.pem 2048 Generating RSA private key, 2048 bit long modulus .........+++ .........+++ e is 65537 (0x10001) + openssl req -x509 -new -nodes -key ca.key.pem -subj /CN=CARoot -days 365 -out ca.cert.pem + openssl genrsa -out server.key.pem 2048 Generating RSA private key, 2048 bit long modulus ..+++ .......................................+++ e is 65537 (0x10001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in server.key.pem -out server.key-pk8.pem -nocrypt + echo '[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + openssl req -new -config server.conf -key server.key.pem -out server.csr.pem -sha256 + openssl x509 -req -in server.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out server.cert.pem -days 365 -extensions v3_ext -extfile server.conf -sha256 Signature ok subject=/CN=server Getting CA Private Key + openssl genrsa -out broker_client.key.pem 2048 Generating RSA private key, 2048 bit long modulus .......+++ ...............+++ e is 65537 (0x10001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in broker_client.key.pem -out broker_client.key-pk8.pem -nocrypt + openssl req -new -subj /CN=broker_client -key broker_client.key.pem -out broker_client.csr.pem -sha256 + openssl x509 -req -in broker_client.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out broker_client.cert.pem -days 365 -sha256 Signature ok subject=/CN=broker_client Getting CA Private Key + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/region_merge/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/region_merge/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/region_merge/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/region_merge/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/region_merge/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/region_merge/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' + echo ' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/region_merge/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/region_merge/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/region_merge/ca.cert.pem' ++ date + echo '[Mon May 6 16:59:41 CST 2024] <<<<<< START pulsar cluster in mtls mode in region_merge case >>>>>>' [Mon May 6 16:59:41 CST 2024] <<<<<< START pulsar cluster in mtls mode in region_merge case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6651 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/region_merge/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/region_merge/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/region_merge/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 check data successfully Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public wait process cdc.test exit for 1-th time... Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public wait process cdc.test exit for 3-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/open_protocol_claim_check/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:59:42 CST 2024] <<<<<< run test case open_protocol_claim_check success! >>>>>> cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:59:43 CST 2024] <<<<<< run test case ddl_puller_lag success! >>>>>> + nc -z localhost 6651 + i=2 + '[' 2 -gt 20 ']' + sleep 2 Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) public/default ++ date + echo '[Mon May 6 16:59:44 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:59:44 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.12792.out cli changefeed create --start-ts=449573282913714178 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-partition-table-test-15151?protocol=canal-json&enable-tidb-extension=true' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Tenant not found Reason: Tenant not found + i=1 + '[' 1 -gt 20 ']' + sleep 2 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... Create changefeed successfully! ID: a3b6cd99-74bc-4d0f-b696-1ebdc8b01219 Info: {"upstream_id":7365808601977377040,"namespace":"default","id":"a3b6cd99-74bc-4d0f-b696-1ebdc8b01219","sink_uri":"pulsar://127.0.0.1:6650/ticdc-partition-table-test-15151?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:59:45.081570681+08:00","start_ts":449573282913714178,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573282913714178,"checkpoint_ts":449573282913714178,"checkpoint_time":"2024-05-06 16:59:33.029"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + nc -z localhost 6651 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/canal_json_storage_basic/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:59:46 CST 2024] <<<<<< run test case canal_json_storage_basic success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... + set +x [Mon May 6 16:59:46 CST 2024] <<<<<< START Pulsar consumer in partition_table case >>>>>> + /usr/local/pulsar/bin/pulsar-admin namespaces list public wait process cdc.test exit for 3-th time... start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:59:47 CST 2024] <<<<<< run test case owner_resign success! >>>>>> public/default ++ date + echo '[Mon May 6 16:59:48 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:59:48 CST 2024] <<<<<< pulsar is ready >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: b43e9a81-e9ef-4f92-9381-51382fd0a9bd Info: {"upstream_id":7365808629334094005,"namespace":"default","id":"b43e9a81-e9ef-4f92-9381-51382fd0a9bd","sink_uri":"pulsar+ssl://127.0.0.1:6651/ticdc-region-merge-test-28739?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:59:48.977630242+08:00","start_ts":449573286923206658,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"tls-trust-certs-file-path":"/tmp/tidb_cdc_test/region_merge/ca.cert.pem","connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30,"auth-tls-certificate-path":"/tmp/tidb_cdc_test/region_merge/broker_client.cert.pem","auth-tls-private-key-path":"/tmp/tidb_cdc_test/region_merge/broker_client.key-pk8.pem"},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573286923206658,"checkpoint_ts":449573286923206658,"checkpoint_time":"2024-05-06 16:59:48.324"} [Mon May 6 16:59:49 CST 2024] <<<<<< START Pulsar consumer in region_merge case >>>>>> split_and_random_merge scale: 20 public/default ++ date + echo '[Mon May 6 16:59:49 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:59:49 CST 2024] <<<<<< pulsar is ready >>>>>> [Mon May 6 16:59:49 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.68586860.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3485c2d40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-q6kw-vxk5b, pid:5248, start at 2024-05-06 16:59:48.307701345 +0800 CST m=+6.112518433 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:48.315 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:48.277 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:48.277 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3485c2d40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-q6kw-vxk5b, pid:5248, start at 2024-05-06 16:59:48.307701345 +0800 CST m=+6.112518433 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:48.315 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:48.277 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:48.277 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3485c7840006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-q6kw-vxk5b, pid:5331, start at 2024-05-06 16:59:48.587429423 +0800 CST m=+6.329506546 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:48.597 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:48.577 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:48.577 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: pulsar... <<================= Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3485d16c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7wx1-4ndrn, pid:4913, start at 2024-05-06 16:59:49.233408899 +0800 CST m=+5.391812011 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:49.240 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:49.211 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:49.211 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3485d16c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7wx1-4ndrn, pid:4913, start at 2024-05-06 16:59:49.233408899 +0800 CST m=+5.391812011 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:49.240 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:49.211 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:49.211 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3485d2140015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7wx1-4ndrn, pid:4990, start at 2024-05-06 16:59:49.295061723 +0800 CST m=+5.402659820 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:49.302 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:49.303 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:49.303 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:52 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45d6d482-51eb-4242-894e-8db407db0f01 {"id":"45d6d482-51eb-4242-894e-8db407db0f01","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985989} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d210dc8fe 45d6d482-51eb-4242-894e-8db407db0f01 /tidb/cdc/default/default/upstream/7365808630844971230 {"id":7365808630844971230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45d6d482-51eb-4242-894e-8db407db0f01 {"id":"45d6d482-51eb-4242-894e-8db407db0f01","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985989} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d210dc8fe 45d6d482-51eb-4242-894e-8db407db0f01 /tidb/cdc/default/default/upstream/7365808630844971230 {"id":7365808630844971230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45d6d482-51eb-4242-894e-8db407db0f01 {"id":"45d6d482-51eb-4242-894e-8db407db0f01","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985989} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d210dc8fe 45d6d482-51eb-4242-894e-8db407db0f01 /tidb/cdc/default/default/upstream/7365808630844971230 {"id":7365808630844971230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ date + echo '[Mon May 6 16:59:52 CST 2024] <<<<<< START pulsar cluster in changefeed_pause_resume case >>>>>>' [Mon May 6 16:59:52 CST 2024] <<<<<< START pulsar cluster in changefeed_pause_resume case >>>>>> + workdir=/tmp/tidb_cdc_test/changefeed_pause_resume + cluster_type=normal + cd /tmp/tidb_cdc_test/changefeed_pause_resume + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/changefeed_pause_resume/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/changefeed_pause_resume/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/changefeed_pause_resume/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/changefeed_pause_resume/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/changefeed_pause_resume/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/changefeed_pause_resume/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/changefeed_pause_resume/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:59:52 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_pause_resume case >>>>>>' [Mon May 6 16:59:52 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_pause_resume case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/changefeed_pause_resume/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/changefeed_pause_resume/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/changefeed_pause_resume/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 [Mon May 6 16:59:53 CST 2024] <<<<<< START Pulsar consumer in changefeed_reconstruct case >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "readallfields"="true" "mysql.user"="root" "updateproportion"="0" "workload"="core" "threadcount"="4" "mysql.host"="127.0.0.1" "mysql.db"="changefeed_reconstruct" "mysql.port"="4000" "operationcount"="0" "recordcount"="50" "readproportion"="0" "dotransactions"="false" "insertproportion"="0" "requestdistribution"="uniform" "scanproportion"="0" ********************************************** Run finished, takes 25.188688ms INSERT - Takes(s): 0.0, Count: 48, OPS: 2406.3, Avg(us): 2014, Min(us): 1098, Max(us): 5115, 95th(us): 6000, 99th(us): 6000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/run.sh using Sink-Type: pulsar... <<================= [Mon May 6 16:59:53 CST 2024] <<<<<< run test case canal_json_storage_partition_table success! >>>>>> + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.6383.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc already exists) + set +x + tso='449573288558460929 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573288558460929 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:59:56 CST 2024] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.64276429.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) *************************** 1. row *************************** count(distinct region_id): 1 start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_tables_ddl/run.sh using Sink-Type: pulsar... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide public/default check diff successfully table partition_table.t exists table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ date + echo '[Mon May 6 16:59:59 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 16:59:59 CST 2024] <<<<<< pulsar is ready >>>>>> [Mon May 6 16:59:59 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.70647066.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCDAAE132834FC < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 08:59:59 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact wait process 6863 exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:59 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/26206859-3da9-4f10-8803-35a6d6b8bca0 {"id":"26206859-3da9-4f10-8803-35a6d6b8bca0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985996} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21495dd7 26206859-3da9-4f10-8803-35a6d6b8bca0 /tidb/cdc/default/default/upstream/7365808694752089857 {"id":7365808694752089857,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/26206859-3da9-4f10-8803-35a6d6b8bca0 {"id":"26206859-3da9-4f10-8803-35a6d6b8bca0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985996} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21495dd7 26206859-3da9-4f10-8803-35a6d6b8bca0 /tidb/cdc/default/default/upstream/7365808694752089857 {"id":7365808694752089857,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/26206859-3da9-4f10-8803-35a6d6b8bca0 {"id":"26206859-3da9-4f10-8803-35a6d6b8bca0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985996} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21495dd7 26206859-3da9-4f10-8803-35a6d6b8bca0 /tidb/cdc/default/default/upstream/7365808694752089857 {"id":7365808694752089857,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 16:59:59 CST 2024] <<<<<< START pulsar cluster in ddl_attributes case >>>>>>' [Mon May 6 16:59:59 CST 2024] <<<<<< START pulsar cluster in ddl_attributes case >>>>>> + workdir=/tmp/tidb_cdc_test/ddl_attributes + cluster_type=normal + cd /tmp/tidb_cdc_test/ddl_attributes + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_attributes/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/ddl_attributes/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/ddl_attributes/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_attributes/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/ddl_attributes/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/ddl_attributes/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/ddl_attributes/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/ddl_attributes/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_attributes/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/ddl_attributes/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/ddl_attributes/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/ddl_attributes/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 16:59:59 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_attributes case >>>>>>' [Mon May 6 16:59:59 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_attributes case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/ddl_attributes/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/ddl_attributes/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/ddl_attributes/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 Bucket 's3://logbucket/' created Exiting on signal: INTERRUPT table partition_table.t2 not exists for 2-th check, retry later wait process 6863 exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process 6863 exit for 3-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (6863) - No such process wait process 6863 exit for 4-th time... process 6863 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 17:00:00 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.72447246.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348665480014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8, pid:12590, start at 2024-05-06 16:59:58.71181792 +0800 CST m=+5.922805768 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:58.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:58.723 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:58.723 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348665480014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8, pid:12590, start at 2024-05-06 16:59:58.71181792 +0800 CST m=+5.922805768 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:58.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:58.723 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:58.723 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348666f40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-9btd-sthr8, pid:12672, start at 2024-05-06 16:59:58.824393858 +0800 CST m=+5.979818368 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:58.834 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:58.831 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:58.831 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 split_and_random_merge scale: 40 Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.t2 not exists for 3-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 09:00:02 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e3f30090-be18-4b9b-87f9-859d98f770c2 {"id":"e3f30090-be18-4b9b-87f9-859d98f770c2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985999} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d2142fbe9 e3f30090-be18-4b9b-87f9-859d98f770c2 /tidb/cdc/default/default/upstream/7365808693106457009 {"id":7365808693106457009,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e3f30090-be18-4b9b-87f9-859d98f770c2 {"id":"e3f30090-be18-4b9b-87f9-859d98f770c2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985999} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d2142fbe9 e3f30090-be18-4b9b-87f9-859d98f770c2 /tidb/cdc/default/default/upstream/7365808693106457009 {"id":7365808693106457009,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e3f30090-be18-4b9b-87f9-859d98f770c2 {"id":"e3f30090-be18-4b9b-87f9-859d98f770c2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714985999} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d2142fbe9 e3f30090-be18-4b9b-87f9-859d98f770c2 /tidb/cdc/default/default/upstream/7365808693106457009 {"id":7365808693106457009,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.14012.out cli tso query --pd=http://127.0.0.1:2379 [Mon May 6 17:00:03 CST 2024] <<<<<< START Pulsar consumer in changefeed_pause_resume case >>>>>> + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public \033[0;36m<<< Run all test success >>>\033[0m table changefeed_pause_resume.t1 not exists for 1-th check, retry later [Pipeline] } table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 09:00:03 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/a945a65f-5cfd-469e-b623-81b0c58587e6 {UpstreamID:7365808630844971230 Namespace:default ID:a945a65f-5cfd-469e-b623-81b0c58587e6 SinkURI:pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-13465?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 16:59:53.242496266 +0800 CST StartTs:449573288020279307 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0015d85a0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573288204042247} {CheckpointTs:449573289488547847 MinTableBarrierTs:449573290799267846 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449573289488547847, checkpointTs: 449573289488547847, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a0c2863a-5de2-4874-b1e4-478f3c0cb54e {"id":"a0c2863a-5de2-4874-b1e4-478f3c0cb54e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986001} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d210dc9b0 a0c2863a-5de2-4874-b1e4-478f3c0cb54e /tidb/cdc/default/default/changefeed/info/a945a65f-5cfd-469e-b623-81b0c58587e6 {"upstream-id":7365808630844971230,"namespace":"default","changefeed-id":"a945a65f-5cfd-469e-b623-81b0c58587e6","sink-uri":"pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-13465?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T16:59:53.242496266+08:00","start-ts":449573288020279307,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573288204042247} /tidb/cdc/default/default/changefeed/status/a945a65f-5cfd-469e-b623-81b0c58587e6 {"checkpoint-ts":449573289488547847,"min-table-barrier-ts":449573290799267846,"admin-job-type":0} /tidb/cdc/default/default/task/position/a0c2863a-5de2-4874-b1e4-478f3c0cb54e/a945a65f-5cfd-469e-b623-81b0c58587e6 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808630844971230 {"id":7365808630844971230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/a945a65f-5cfd-469e-b623-81b0c58587e6 {UpstreamID:7365808630844971230 Namespace:default ID:a945a65f-5cfd-469e-b623-81b0c58587e6 SinkURI:pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-13465?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 16:59:53.242496266 +0800 CST StartTs:449573288020279307 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0015d85a0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573288204042247} {CheckpointTs:449573289488547847 MinTableBarrierTs:449573290799267846 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449573289488547847, checkpointTs: 449573289488547847, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a0c2863a-5de2-4874-b1e4-478f3c0cb54e {"id":"a0c2863a-5de2-4874-b1e4-478f3c0cb54e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986001} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d210dc9b0 a0c2863a-5de2-4874-b1e4-478f3c0cb54e /tidb/cdc/default/default/changefeed/info/a945a65f-5cfd-469e-b623-81b0c58587e6 {"upstream-id":7365808630844971230,"namespace":"default","changefeed-id":"a945a65f-5cfd-469e-b623-81b0c58587e6","sink-uri":"pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-13465?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T16:59:53.242496266+08:00","start-ts":449573288020279307,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573288204042247} /tidb/cdc/default/default/changefeed/status/a945a65f-5cfd-469e-b623-81b0c58587e6 {"checkpoint-ts":449573289488547847,"min-table-barrier-ts":449573290799267846,"admin-job-type":0} /tidb/cdc/default/default/task/position/a0c2863a-5de2-4874-b1e4-478f3c0cb54e/a945a65f-5cfd-469e-b623-81b0c58587e6 {"checkpoint+ grep -q 'failed to get info:' -ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808630844971230 {"id":7365808630844971230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/a945a65f-5cfd-469e-b623-81b0c58587e6 {UpstreamID:7365808630844971230 Namespace:default ID:a945a65f-5cfd-469e-b623-81b0c58587e6 SinkURI:pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-13465?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 16:59:53.242496266 +0800 CST StartTs:449573288020279307 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0015d85a0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573288204042247} {CheckpointTs:449573289488547847 MinTableBarrierTs:449573290799267846 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449573289488547847, checkpointTs: 449573289488547847, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a0c2863a-5de2-4874-b1e4-478f3c0cb54e {"id":"a0c2863a-5de2-4874-b1e4-478f3c0cb54e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986001} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d210dc9b0 a0c2863a-5de2-4874-b1e4-478f3c0cb54e /tidb/cdc/default/default/changefeed/info/a945a65f-5cfd-469e-b623-81b0c58587e6 {"upstream-id":7365808630844971230,"namespace":"default","changefeed-id":"a945a65f-5cfd-469e-b623-81b0c58587e6","sink-uri":"pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-13465?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T16:59:53.242496266+08:00","start-ts":449573288020279307,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573288204042247} /tidb/cdc/default/default/changefeed/status/a945a65f-5cfd-469e-b623-81b0c58587e6 {"checkpoint-ts":449573289488547847,"min-table-barrier-ts":449573290799267846,"admin-job-type":0} /tidb/cdc/default/default/task/position/a0c2863a-5de2-4874-b1e4-478f3c0cb54e/a945a65f-5cfd-469e-b623-81b0c58587e6 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808630844971230 {"id":7365808630844971230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449573290907795457 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573290907795457 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 17:00:05 CST 2024] <<<<<< START cdc server in resourcecontrol case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.1405614058.out server --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resourcecontrol/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 not exists for 1-th check, retry later "id": "a0c2863a-5de2-4874-b1e4-478f3c0cb54e", "cluster-id": "default" run task successfully table partition_table.finish_mark not exists for 2-th check, retry later Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc already exists) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) public/default VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3486c0680005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7txd-3l8p2, pid:4539, start at 2024-05-06 17:00:04.519657594 +0800 CST m=+10.687160563 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:04.556 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:04.556 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:04.556 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3486c0680005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7txd-3l8p2, pid:4539, start at 2024-05-06 17:00:04.519657594 +0800 CST m=+10.687160563 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:04.556 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:04.556 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:04.556 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3486c4180001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-7txd-3l8p2, pid:4574, start at 2024-05-06 17:00:04.761375911 +0800 CST m=+10.806412603 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:04.781 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:04.791 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:04.791 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } capture_id: a0c2863a-5de2-4874-b1e4-478f3c0cb54e check_processor_table_count http://127.0.0.1:2379 a945a65f-5cfd-469e-b623-81b0c58587e6 a0c2863a-5de2-4874-b1e4-478f3c0cb54e 1 [Pipeline] // container [Pipeline] } ++ date + echo '[Mon May 6 17:00:06 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 17:00:06 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.6886.out cli changefeed create --start-ts=449573288558460929 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-ddl-attributes-test-7060?protocol=canal-json&enable-tidb-extension=true' [Pipeline] // withEnv [Pipeline] } [Pipeline] // node run task successfully [Pipeline] } Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table partition_table.finish_mark not exists for 3-th check, retry later check_processor_table_count http://127.0.0.1:2379 a945a65f-5cfd-469e-b623-81b0c58587e6 a0c2863a-5de2-4874-b1e4-478f3c0cb54e 0 Create changefeed successfully! ID: 03f0cba6-c42e-49be-9575-9d4661a01644 Info: {"upstream_id":7365808694752089857,"namespace":"default","id":"03f0cba6-c42e-49be-9575-9d4661a01644","sink_uri":"pulsar://127.0.0.1:6650/ticdc-ddl-attributes-test-7060?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T17:00:08.028145946+08:00","start_ts":449573288558460929,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573288558460929,"checkpoint_ts":449573288558460929,"checkpoint_time":"2024-05-06 16:59:54.562"} PASS table count 1 does equal to expected count 0 run task failed 1-th time, retry later table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later *************************** 1. row *************************** count(distinct region_id): 3 coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 09:00:08 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/315e0e79-0f02-41fe-bbba-805eabd320eb {"id":"315e0e79-0f02-41fe-bbba-805eabd320eb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986005} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d216a75d3 315e0e79-0f02-41fe-bbba-805eabd320eb /tidb/cdc/default/default/upstream/7365808739483356527 {"id":7365808739483356527,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/315e0e79-0f02-41fe-bbba-805eabd320eb {"id":"315e0e79-0f02-41fe-bbba-805eabd320eb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986005} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d216a75d3 315e0e79-0f02-41fe-bbba-805eabd320eb /tidb/cdc/default/default/upstream/7365808739483356527 {"id":7365808739483356527,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/315e0e79-0f02-41fe-bbba-805eabd320eb {"id":"315e0e79-0f02-41fe-bbba-805eabd320eb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986005} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d216a75d3 315e0e79-0f02-41fe-bbba-805eabd320eb /tidb/cdc/default/default/upstream/7365808739483356527 {"id":7365808739483356527,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 17:00:08 CST 2024] <<<<<< START pulsar cluster in resourcecontrol case >>>>>>' [Mon May 6 17:00:08 CST 2024] <<<<<< START pulsar cluster in resourcecontrol case >>>>>> + workdir=/tmp/tidb_cdc_test/resourcecontrol + cluster_type=normal + cd /tmp/tidb_cdc_test/resourcecontrol + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/resourcecontrol/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/resourcecontrol/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/resourcecontrol/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/resourcecontrol/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/resourcecontrol/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/resourcecontrol/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/resourcecontrol/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/resourcecontrol/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/resourcecontrol/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/resourcecontrol/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/resourcecontrol/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/resourcecontrol/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 17:00:08 CST 2024] <<<<<< START pulsar cluster in normal mode in resourcecontrol case >>>>>>' [Mon May 6 17:00:08 CST 2024] <<<<<< START pulsar cluster in normal mode in resourcecontrol case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/resourcecontrol/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/resourcecontrol/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/resourcecontrol/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3486edec0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-g720-bjdgs, pid:8195, start at 2024-05-06 17:00:07.427968201 +0800 CST m=+5.505884897 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:07.436 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:07.419 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:07.419 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3486edec0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-g720-bjdgs, pid:8195, start at 2024-05-06 17:00:07.427968201 +0800 CST m=+5.505884897 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:07.436 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:07.419 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:07.419 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3486ef680003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-g720-bjdgs, pid:8272, start at 2024-05-06 17:00:07.517181624 +0800 CST m=+5.536939291 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:07.525 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:07.514 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:07.514 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.5910.out cli tso query --pd=http://127.0.0.1:2379 table partition_table.finish_mark not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_processor_table_count http://127.0.0.1:2379 a945a65f-5cfd-469e-b623-81b0c58587e6 a0c2863a-5de2-4874-b1e4-478f3c0cb54e 0 table changefeed_pause_resume.t3 exists + set +x [Mon May 6 17:00:09 CST 2024] <<<<<< START Pulsar consumer in ddl_attributes case >>>>>> run task successfully + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 [Mon May 6 17:00:10 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.96089610.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449573292371083265 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573292371083265 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "updateproportion"="0" "insertproportion"="0" "threadcount"="4" "readproportion"="0" "operationcount"="0" "mysql.host"="127.0.0.1" "workload"="core" "readallfields"="true" "mysql.db"="changefeed_reconstruct" "dotransactions"="false" "mysql.user"="root" "mysql.port"="4000" "scanproportion"="0" "recordcount"="50" "requestdistribution"="uniform" ********************************************** Run finished, takes 25.946611ms INSERT - Takes(s): 0.0, Count: 48, OPS: 2300.5, Avg(us): 2049, Min(us): 1053, Max(us): 4968, 95th(us): 5000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) try a VALID cdc server command [Mon May 6 17:00:11 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.59335935.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later table partition_table.finish_mark not exists for 5-th check, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public split_and_random_merge scale: 80 table changefeed_reconstruct.usertable exists check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34873e740018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-1bd3-dwfrg, pid:4753, start at 2024-05-06 17:00:12.613049562 +0800 CST m=+5.392735412 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:12.620 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:12.623 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:12.623 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 6-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 09:00:13 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3024cdfe-491e-4411-b2fc-e0ffc3befc71 {"id":"3024cdfe-491e-4411-b2fc-e0ffc3befc71","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986011} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d219402c9 3024cdfe-491e-4411-b2fc-e0ffc3befc71 /tidb/cdc/default/default/upstream/7365808775016593632 {"id":7365808775016593632,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3024cdfe-491e-4411-b2fc-e0ffc3befc71 {"id":"3024cdfe-491e-4411-b2fc-e0ffc3befc71","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986011} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d219402c9 3024cdfe-491e-4411-b2fc-e0ffc3befc71 /tidb/cdc/default/default/upstream/7365808775016593632 {"id":7365808775016593632,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3024cdfe-491e-4411-b2fc-e0ffc3befc71 {"id":"3024cdfe-491e-4411-b2fc-e0ffc3befc71","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986011} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d219402c9 3024cdfe-491e-4411-b2fc-e0ffc3befc71 /tidb/cdc/default/default/upstream/7365808775016593632 {"id":7365808775016593632,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Mon May 6 17:00:13 CST 2024] <<<<<< START pulsar cluster in ddl_only_block_related_table case >>>>>>' [Mon May 6 17:00:13 CST 2024] <<<<<< START pulsar cluster in ddl_only_block_related_table case >>>>>> + workdir=/tmp/tidb_cdc_test/ddl_only_block_related_table + cluster_type=normal + cd /tmp/tidb_cdc_test/ddl_only_block_related_table + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_only_block_related_table/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/ddl_only_block_related_table/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/ddl_only_block_related_table/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_only_block_related_table/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/ddl_only_block_related_table/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/ddl_only_block_related_table/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/ddl_only_block_related_table/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/ddl_only_block_related_table/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_only_block_related_table/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/ddl_only_block_related_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/ddl_only_block_related_table/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 17:00:13 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_only_block_related_table case >>>>>>' [Mon May 6 17:00:13 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_only_block_related_table case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/ddl_only_block_related_table/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 wait process cdc.test exit for 1-th time... check diff failed 2-th time, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 09:00:14 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f0e8293f-764a-4841-9737-a59816db4062 {"id":"f0e8293f-764a-4841-9737-a59816db4062","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986012} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d216ddbf0 f0e8293f-764a-4841-9737-a59816db4062 /tidb/cdc/default/default/upstream/7365808742100477725 {"id":7365808742100477725,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f0e8293f-764a-4841-9737-a59816db4062 {"id":"f0e8293f-764a-4841-9737-a59816db4062","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986012} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d216ddbf0 f0e8293f-764a-4841-9737-a59816db4062 /tidb/cdc/default/default/upstream/7365808742100477725 {"id":7365808742100477725,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f0e8293f-764a-4841-9737-a59816db4062 {"id":"f0e8293f-764a-4841-9737-a59816db4062","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986012} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d216ddbf0 f0e8293f-764a-4841-9737-a59816db4062 /tidb/cdc/default/default/upstream/7365808742100477725 {"id":7365808742100477725,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 17:00:15 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34873e740018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-1bd3-dwfrg, pid:4753, start at 2024-05-06 17:00:12.613049562 +0800 CST m=+5.392735412 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:12.620 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:12.623 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:12.623 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348740bc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-1bd3-dwfrg, pid:4843, start at 2024-05-06 17:00:12.753162013 +0800 CST m=+5.477170847 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:12.760 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:12.719 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:12.719 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table partition_table.finish_mark not exists for 7-th check, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 check diff failed 3-th time, retry later Tenant not found Reason: Tenant not found + i=1 + '[' 1 -gt 20 ']' + sleep 2 table partition_table.finish_mark not exists for 8-th check, retry later + nc -z localhost 6650 + i=3 + '[' 3 -gt 20 ']' + sleep 2 ++ date + echo '[Mon May 6 17:00:17 CST 2024] <<<<<< START pulsar cluster in processor_etcd_worker_delay case >>>>>>' [Mon May 6 17:00:17 CST 2024] <<<<<< START pulsar cluster in processor_etcd_worker_delay case >>>>>> + workdir=/tmp/tidb_cdc_test/processor_etcd_worker_delay + cluster_type=normal + cd /tmp/tidb_cdc_test/processor_etcd_worker_delay + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/processor_etcd_worker_delay/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/processor_etcd_worker_delay/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/processor_etcd_worker_delay/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_etcd_worker_delay/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/processor_etcd_worker_delay/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/processor_etcd_worker_delay/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/processor_etcd_worker_delay/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/processor_etcd_worker_delay/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_etcd_worker_delay/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/processor_etcd_worker_delay/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/processor_etcd_worker_delay/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/processor_etcd_worker_delay/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Mon May 6 17:00:17 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_etcd_worker_delay case >>>>>>' [Mon May 6 17:00:17 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_etcd_worker_delay case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/processor_etcd_worker_delay/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 check diff failed 4-th time, retry later table partition_table.finish_mark exists check diff successfully + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + /usr/local/pulsar/bin/pulsar-admin namespaces list public check diff failed 5-th time, retry later + nc -z localhost 6650 + i=4 + '[' 4 -gt 20 ']' + sleep 2 wait process cdc.test exit for 1-th time... *************************** 1. row *************************** count(distinct region_id): 1 wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 17:00:21 CST 2024] <<<<<< run test case partition_table success! >>>>>> + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public public/default ++ date + echo '[Mon May 6 17:00:22 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 17:00:22 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.14606.out cli changefeed create --start-ts=449573290907795457 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-resourcecontrol-test-26547?protocol=canal-json&enable-tidb-extension=true' + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public check diff successfully Create changefeed successfully! ID: 834a8719-ba36-4173-8a2f-85f9cec26396 Info: {"upstream_id":7365808739483356527,"namespace":"default","id":"834a8719-ba36-4173-8a2f-85f9cec26396","sink_uri":"pulsar://127.0.0.1:6650/ticdc-resourcecontrol-test-26547?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T17:00:23.187871599+08:00","start_ts":449573290907795457,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573290907795457,"checkpoint_ts":449573290907795457,"checkpoint_time":"2024-05-06 17:00:03.524"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... public/default ++ date + echo '[Mon May 6 17:00:24 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 17:00:24 CST 2024] <<<<<< pulsar is ready >>>>>> [Mon May 6 17:00:24 CST 2024] <<<<<< START cdc server in processor_etcd_worker_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/orchestrator/ProcessorEtcdDelay=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_etcd_worker_delay.65856587.out server --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 public/default ++ date + echo '[Mon May 6 17:00:24 CST 2024] <<<<<< pulsar is ready >>>>>>' [Mon May 6 17:00:24 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.10076.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-common-1-test-6098?protocol=canal-json&enable-tidb-extension=true' -c=ddl-only-block-related-table check diff failed 1-th time, retry later + set +x [Mon May 6 17:00:24 CST 2024] <<<<<< START Pulsar consumer in resourcecontrol case >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table region_merge.t1 exists check diff failed 1-th time, retry later Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7365808775016593632,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"pulsar://127.0.0.1:6650/ticdc-common-1-test-6098?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T17:00:25.432227871+08:00","start_ts":449573296463937542,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573296463937542,"checkpoint_ts":449573296463937542,"checkpoint_time":"2024-05-06 17:00:24.719"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 2-th time, retry later + set +x [Mon May 6 17:00:26 CST 2024] <<<<<< START Pulsar consumer in ddl_only_block_related_table case >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 09:00:27 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47ce92f9-17cf-430a-9980-7761cb26545e {"id":"47ce92f9-17cf-430a-9980-7761cb26545e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986024} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21a4d1e6 47ce92f9-17cf-430a-9980-7761cb26545e /tidb/cdc/default/default/upstream/7365808802090676977 {"id":7365808802090676977,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47ce92f9-17cf-430a-9980-7761cb26545e {"id":"47ce92f9-17cf-430a-9980-7761cb26545e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986024} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21a4d1e6 47ce92f9-17cf-430a-9980-7761cb26545e /tidb/cdc/default/default/upstream/7365808802090676977 {"id":7365808802090676977,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47ce92f9-17cf-430a-9980-7761cb26545e {"id":"47ce92f9-17cf-430a-9980-7761cb26545e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986024} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21a4d1e6 47ce92f9-17cf-430a-9980-7761cb26545e /tidb/cdc/default/default/upstream/7365808802090676977 {"id":7365808802090676977,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table resourcecontrol.finish_mark not exists for 1-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later check diff successfully [Mon May 6 17:00:27 CST 2024] <<<<<< START Pulsar consumer in processor_etcd_worker_delay case >>>>>> wait process cdc.test exit for 1-th time... table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark not exists for 1-th check, retry later table ddl_attributes.finish_mark not exists for 2-th check, retry later check diff failed 3-th time, retry later wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... wait process cdc.test exit for 3-th time... table resourcecontrol.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 17:00:29 CST 2024] <<<<<< run test case region_merge success! >>>>>> table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later table ddl_attributes.finish_mark not exists for 3-th check, retry later wait process cdc.test exit for 1-th time... check diff failed 4-th time, retry later table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later table ddl_attributes.finish_mark not exists for 4-th check, retry later check diff failed 5-th time, retry later cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Mon May 6 17:00:32 CST 2024] <<<<<< run test case resourcecontrol success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_only_block_related_table.finish_mark not exists for 4-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } table processor_delay.t1 exists table processor_delay.t2 not exists for 1-th check, retry later [Pipeline] // container table ddl_attributes.finish_mark not exists for 5-th check, retry later [Pipeline] } check diff successfully [Pipeline] // withEnv Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table ddl_only_block_related_table.finish_mark exists wait process 9613 exit for 1-th time... table processor_delay.t2 exists table processor_delay.t3 not exists for 1-th check, retry later table ddl_attributes.finish_mark not exists for 6-th check, retry later check diff failed 1-th time, retry later wait process 9613 exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process 9613 exit for 3-th time... valid ~~~ running cdc Failed to start cdc, the usage tips should be printed 1st test case cdc_server_tips success! try an INVALID cdc server command [Mon May 6 17:00:34 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ true != \n\o ]] + set +x + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.59965998.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default --pd None /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (9613) - No such process wait process 9613 exit for 4-th time... process 9613 already exit [Mon May 6 17:00:37 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.1022410226.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_attributes.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... table processor_delay.t3 exists table processor_delay.t4 not exists for 1-th check, retry later check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 17:00:40 CST 2024] <<<<<< run test case ddl_attributes success! >>>>>> table processor_delay.t4 exists table processor_delay.t5 not exists for 1-th check, retry later check diff failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 09:00:40 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365808775016593632 Namespace:default ID:ddl-only-block-related-table SinkURI:pulsar://127.0.0.1:6650/ticdc-common-1-test-6098?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 17:00:25.432227871 +0800 CST StartTs:449573296463937542 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b1c3f0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573296647438341} {CheckpointTs:449573300474740738 MinTableBarrierTs:449573300474740740 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449573300474740738, checkpointTs: 449573300474740738, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449573300474740738, checkpointTs: 449573300474740738, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449573300474740738, checkpointTs: 449573300474740738, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449573300474740738, checkpointTs: 449573300474740738, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74d17609-0cbe-4b2c-8f73-c6b2c2769a75 {"id":"74d17609-0cbe-4b2c-8f73-c6b2c2769a75","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986037} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d219403db 74d17609-0cbe-4b2c-8f73-c6b2c2769a75 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365808775016593632,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"pulsar://127.0.0.1:6650/ticdc-common-1-test-6098?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T17:00:25.432227871+08:00","start-ts":449573296463937542,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573296647438341} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449573300474740738,"min-table-barrier-ts":449573300474740740,"admin-job-type":0} /tidb/cdc/default/default/task/position/74d17609-0cbe-4b2c-8f73-c6b2c2769a75/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808775016593632 {"id":7365808775016593632,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365808775016593632 Namespace:default ID:ddl-only-block-related-table SinkURI:pulsar://127.0.0.1:6650/ticdc-common-1-test-6098?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 17:00:25.432227871 +0800 CST StartTs:449573296463937542 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b1c3f0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573296647438341} {CheckpointTs:449573300474740738 MinTableBarrierTs:449573300474740740 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449573300474740738, checkpointTs: 449573300474740738, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449573300474740738, checkpointTs: 449573300474740738, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449573300474740738, checkpointTs: 449573300474740738, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449573300474740738, checkpointTs: 449573300474740738, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74d17609-0cbe-4b2c-8f73-c6b2c2769a75 {"id":"74d17609-0cbe-4b2c-8f73-c6b2c2769a75","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986037} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d219403db 74d17609-0cbe-4b2c-8f73-c6b2c2769a75 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365808775016593632,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"pulsar://127.0.0.1:6650/ticdc-common-1-test-6098?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T17:00:25.432227871+08:00","start-ts":449573296463937542,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573296647438341} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449573300474740738,"min-table-barrier-ts":449573300474740740,"admin-job-type":0} /tidb/cdc/default/default/task/position/74d17609-0cbe-4b2c-8f73-c6b2c2769a75/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808775016593632 {"id":7365808775016593632,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365808775016593632 Namespace:default ID:ddl-only-block-related-table SinkURI:pulsar://127.0.0.1:6650/ticdc-common-1-test-6098?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-06 17:00:25.432227871 +0800 CST StartTs:449573296463937542 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b1c3f0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573296647438341} {CheckpointTs:449573300474740738 MinTableBarrierTs:449573300474740740 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449573300474740738, checkpointTs: 449573300474740738, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449573300474740738, checkpointTs: 449573300474740738, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449573300474740738, checkpointTs: 449573300474740738, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449573300474740738, checkpointTs: 449573300474740738, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74d17609-0cbe-4b2c-8f73-c6b2c2769a75 {"id":"74d17609-0cbe-4b2c-8f73-c6b2c2769a75","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986037} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d219403db 74d17609-0cbe-4b2c-8f73-c6b2c2769a75 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365808775016593632,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"pulsar://127.0.0.1:6650/ticdc-common-1-test-6098?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-06T17:00:25.432227871+08:00","start-ts":449573296463937542,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-erro+ grep -q 'etcd info' r-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573296647438341} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449573300474740738,"min-table-barrier-ts":449573300474740740,"admin-job-type":0} /tidb/cdc/default/default/task/position/74d17609-0cbe-4b2c-8f73-c6b2c2769a75/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808775016593632 {"id":7365808775016593632,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_ts_not_forward ddl-only-block-related-table VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3488e8e00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-j6m9-tmj32, pid:8750, start at 2024-05-06 17:00:39.905039046 +0800 CST m=+5.446185396 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:39.915 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:39.913 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:39.913 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/common_1/run.sh using Sink-Type: pulsar... <<================= table processor_delay.t5 exists table processor_delay.t6 not exists for 1-th check, retry later check diff failed 4-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3488e8e00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-j6m9-tmj32, pid:8750, start at 2024-05-06 17:00:39.905039046 +0800 CST m=+5.446185396 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:39.915 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:39.913 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:39.913 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3488ea680015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1555-j6m9-tmj32, pid:8835, start at 2024-05-06 17:00:40.001142297 +0800 CST m=+5.487293318 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:40.012 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:40.012 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:40.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table processor_delay.t6 exists table processor_delay.t7 not exists for 1-th check, retry later check diff failed 5-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.10180.out cli tso query --pd=http://127.0.0.1:2379 run task failed 1-th time, retry later table processor_delay.t7 exists table processor_delay.t8 not exists for 1-th check, retry later + set +x + tso='449573301823209473 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573301823209473 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "dotransactions"="false" "mysql.host"="127.0.0.1" "threadcount"="2" "mysql.user"="root" "operationcount"="0" "updateproportion"="0" "readallfields"="true" "mysql.port"="4000" "requestdistribution"="uniform" "scanproportion"="0" "recordcount"="10" "mysql.db"="multi_capture_1" "workload"="core" "readproportion"="0" "insertproportion"="0" ********************************************** Run finished, takes 11.448046ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1480.4, Avg(us): 2114, Min(us): 1292, Max(us): 4564, 95th(us): 5000, 99th(us): 5000 check diff failed 6-th time, retry later ***************** properties ***************** "insertproportion"="0" "mysql.host"="127.0.0.1" "scanproportion"="0" "mysql.user"="root" "recordcount"="10" "operationcount"="0" "readproportion"="0" "workload"="core" "mysql.port"="4000" "readallfields"="true" "dotransactions"="false" "updateproportion"="0" "requestdistribution"="uniform" "threadcount"="2" "mysql.db"="multi_capture_2" ********************************************** Run finished, takes 12.462363ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1234.0, Avg(us): 2413, Min(us): 1502, Max(us): 4260, 95th(us): 5000, 99th(us): 5000 \033[0;36m<<< Run all test success >>>\033[0m ***************** properties ***************** "readallfields"="true" "operationcount"="0" "requestdistribution"="uniform" "mysql.db"="multi_capture_3" "mysql.host"="127.0.0.1" "mysql.user"="root" "readproportion"="0" "mysql.port"="4000" "workload"="core" "threadcount"="2" "dotransactions"="false" "scanproportion"="0" "insertproportion"="0" "updateproportion"="0" "recordcount"="10" ********************************************** Run finished, takes 10.081193ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1724.0, Avg(us): 1948, Min(us): 1203, Max(us): 4268, 95th(us): 5000, 99th(us): 5000 [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } ***************** properties ***************** "mysql.port"="4000" "mysql.db"="multi_capture_4" "insertproportion"="0" "dotransactions"="false" "updateproportion"="0" "mysql.user"="root" "recordcount"="10" "workload"="core" "threadcount"="2" "readallfields"="true" "scanproportion"="0" "mysql.host"="127.0.0.1" "operationcount"="0" "requestdistribution"="uniform" "readproportion"="0" ********************************************** Run finished, takes 9.61855ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1907.9, Avg(us): 1844, Min(us): 1075, Max(us): 4333, 95th(us): 5000, 99th(us): 5000 [Mon May 6 17:00:47 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.1030410306.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node table processor_delay.t8 exists table processor_delay.t9 not exists for 1-th check, retry later check_ts_not_forward ddl-only-block-related-table [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } check_changefeed_state http://127.0.0.1:2379 52ce9f14-aed4-4a5a-b1f2-ddf729b358c7 finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=52ce9f14-aed4-4a5a-b1f2-ddf729b358c7 + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 52ce9f14-aed4-4a5a-b1f2-ddf729b358c7 -s + info='{ "upstream_id": 7365808414695768478, "namespace": "default", "id": "52ce9f14-aed4-4a5a-b1f2-ddf729b358c7", "state": "finished", "checkpoint_tso": 449573298956402694, "checkpoint_time": "2024-05-06 17:00:34.227", "error": null }' + echo '{ "upstream_id": 7365808414695768478, "namespace": "default", "id": "52ce9f14-aed4-4a5a-b1f2-ddf729b358c7", "state": "finished", "checkpoint_tso": 449573298956402694, "checkpoint_time": "2024-05-06 17:00:34.227", "error": null }' { "upstream_id": 7365808414695768478, "namespace": "default", "id": "52ce9f14-aed4-4a5a-b1f2-ddf729b358c7", "state": "finished", "checkpoint_tso": 449573298956402694, "checkpoint_time": "2024-05-06 17:00:34.227", "error": null } ++ echo '{' '"upstream_id":' 7365808414695768478, '"namespace":' '"default",' '"id":' '"52ce9f14-aed4-4a5a-b1f2-ddf729b358c7",' '"state":' '"finished",' '"checkpoint_tso":' 449573298956402694, '"checkpoint_time":' '"2024-05-06' '17:00:34.227",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7365808414695768478, '"namespace":' '"default",' '"id":' '"52ce9f14-aed4-4a5a-b1f2-ddf729b358c7",' '"state":' '"finished",' '"checkpoint_tso":' 449573298956402694, '"checkpoint_time":' '"2024-05-06' '17:00:34.227",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 17:00:44 CST 2024] <<<<<< run test case changefeed_finish success! >>>>>> check diff successfully [Pipeline] // stage [Pipeline] } \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1555/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container table processor_delay.t9 exists table processor_delay.t10 not exists for 1-th check, retry later [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 09:00:50 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1fcb8443-f0c1-45fc-9abb-ee0e6669da43 {"id":"1fcb8443-f0c1-45fc-9abb-ee0e6669da43","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986047} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d220f3c39 1fcb8443-f0c1-45fc-9abb-ee0e6669da43 /tidb/cdc/default/default/upstream/7365808919669544720 {"id":7365808919669544720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1fcb8443-f0c1-45fc-9abb-ee0e6669da43 {"id":"1fcb8443-f0c1-45fc-9abb-ee0e6669da43","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986047} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d220f3c39 1fcb8443-f0c1-45fc-9abb-ee0e6669da43 /tidb/cdc/default/default/upstream/7365808919669544720 {"id":7365808919669544720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1fcb8443-f0c1-45fc-9abb-ee0e6669da43 {"id":"1fcb8443-f0c1-45fc-9abb-ee0e6669da43","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986047} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d220f3c39 1fcb8443-f0c1-45fc-9abb-ee0e6669da43 /tidb/cdc/default/default/upstream/7365808919669544720 {"id":7365808919669544720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 17:00:50 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.1037310375.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check diff failed 1-th time, retry later table processor_delay.t10 exists table processor_delay.t11 not exists for 1-th check, retry later check diff failed 2-th time, retry later Aborted by Jenkins Admin Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 09:00:53 GMT < Content-Length: 1273 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1fcb8443-f0c1-45fc-9abb-ee0e6669da43 {"id":"1fcb8443-f0c1-45fc-9abb-ee0e6669da43","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986047} /tidb/cdc/default/__cdc_meta__/capture/c8c34d28-eacf-43ae-a5e7-fbd8f238450f {"id":"c8c34d28-eacf-43ae-a5e7-fbd8f238450f","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986051} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d220f3c39 1fcb8443-f0c1-45fc-9abb-ee0e6669da43 /tidb/cdc/default/__cdc_meta__/owner/22318f4d220f3c6c c8c34d28-eacf-43ae-a5e7-fbd8f238450f /tidb/cdc/default/default/upstream/7365808919669544720 {"id":7365808919669544720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1fcb8443-f0c1-45fc-9abb-ee0e6669da43 {"id":"1fcb8443-f0c1-45fc-9abb-ee0e6669da43","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986047} /tidb/cdc/default/__cdc_meta__/capture/c8c34d28-eacf-43ae-a5e7-fbd8f238450f {"id":"c8c34d28-eacf-43ae-a5e7-fbd8f238450f","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986051} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d220f3c39 1fcb8443-f0c1-45fc-9abb-ee0e6669da43 /tidb/cdc/default/__cdc_meta__/owner/22318f4d220f3c6c c8c34d28-eacf-43ae-a5e7-fbd8f238450f /tidb/cdc/default/default/upstream/7365808919669544720 {"id":7365808919669544720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1fcb8443-f0c1-45fc-9abb-ee0e6669da43 {"id":"1fcb8443-f0c1-45fc-9abb-ee0e6669da43","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986047} /tidb/cdc/default/__cdc_meta__/capture/c8c34d28-eacf-43ae-a5e7-fbd8f238450f {"id":"c8c34d28-eacf-43ae-a5e7-fbd8f238450f","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714986051} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d220f3c39 1fcb8443-f0c1-45fc-9abb-ee0e6669da43 /tidb/cdc/default/__cdc_meta__/owner/22318f4d220f3c6c c8c34d28-eacf-43ae-a5e7-fbd8f238450f /tidb/cdc/default/default/upstream/7365808919669544720 {"id":7365808919669544720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 17:00:53 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.1042610428.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes invalid ~~~ running cdc Failed to start cdc, the usage tips should be printed 2nd test case cdc_server_tips success! {"level":"warn","ts":1714986054.7388022,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0021bf340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 kill finished with exit code 0 Killing processes kill finished with exit code 0 [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] } Error: [CDC:ErrOwnerNotFound]owner not found /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh: line 1: 10302 Terminated ensure 30 check_ts_not_forward $changefeed_id /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh: line 1: 10505 Terminated ensure 30 check_ts_not_forward $changefeed_id script returned exit code 143 [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // cache [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // cache [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // dir [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // container [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // node [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // podTemplate [Pipeline] // node [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // withEnv [Pipeline] // podTemplate [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G09' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G10' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G11' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G14' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G16' [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // withEnv [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G08' [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G04' [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: 809a91bf-f110-42a7-a372-fb668b71a7a9 Failed in branch Matrix - TEST_GROUP = 'G09' Finished: ABORTED